KIARA_METADATA
¶
find_model_classes: Union[Type, Tuple, Callable]
¶
find_model_classes_api: Union[Type, Tuple, Callable]
¶
Functions¶
DBG(*objects, *, sep=' ', end='\n', file=None, flush=False)
¶
Source code in kiara/__init__.py
def DBG(
*objects: typing.Any,
sep: str = " ",
end: str = "\n",
file: typing.Union[typing.IO[str], None] = None,
flush: bool = False,
):
objs = (
["[green]----------------------------------------------[/green]"]
+ list(objects)
+ ["[green]----------------------------------------------[/green]"]
)
dbg(*objs, sep=sep, end=end, file=file, flush=flush)
dbg(*objects, *, sep=' ', end='\n', file=None, flush=False)
¶
Source code in kiara/__init__.py
def dbg(
*objects: typing.Any,
sep: str = " ",
end: str = "\n",
file: typing.Union[typing.IO[str], None] = None,
flush: bool = False,
):
for obj in objects:
try:
rich_print(obj, sep=sep, end=end, file=file, flush=flush)
except Exception:
rich_print(
f"[green]{obj}[/green]", sep=sep, end=end, file=file, flush=flush
)
get_version()
¶
Return the current version of Kiara.
Source code in kiara/__init__.py
def get_version() -> str:
"""Return the current version of *Kiara*."""
from pkg_resources import DistributionNotFound, get_distribution
try:
# Change here if project is renamed and does not equal the package name
dist_name = __name__
__version__ = get_distribution(dist_name).version
except DistributionNotFound:
try:
version_file = os.path.join(os.path.dirname(__file__), "version.txt")
if os.path.exists(version_file):
with open(version_file, encoding="utf-8") as vf:
__version__ = vf.read()
else:
__version__ = "unknown"
except (Exception):
pass
if __version__ is None:
__version__ = "unknown"
return __version__
Modules¶
context
special
¶
logger
¶
Classes¶
Kiara
¶
The core context of a kiara session.
The Kiara object holds all information related to the current environment the user does works in. This includes:
- available modules, operations & pipelines
- available value data_types
- available metadata schemas
- available data items
- available controller and processor data_types
- misc. configuration options
It's possible to use kiara without ever manually touching the 'Kiara' class, by default all relevant classes and functions
will use a default instance of this class (available via the Kiara.instance() method.
The Kiara class is highly dependent on the Python environment it lives in, because it auto-discovers available sub-classes of its building blocks (modules, value data_types, etc.). So, you can't assume that, for example, a pipeline you create will work the same way (or at all) in a different environment. kiara will always be able to tell you all the details of this environment, though, and it will attach those details to things like data, so there is always a record of how something was created, and in which environment.
Source code in kiara/context/__init__.py
class Kiara(object):
"""The core context of a kiara session.
The `Kiara` object holds all information related to the current environment the user does works in. This includes:
- available modules, operations & pipelines
- available value data_types
- available metadata schemas
- available data items
- available controller and processor data_types
- misc. configuration options
It's possible to use *kiara* without ever manually touching the 'Kiara' class, by default all relevant classes and functions
will use a default instance of this class (available via the `Kiara.instance()` method.
The Kiara class is highly dependent on the Python environment it lives in, because it auto-discovers available sub-classes
of its building blocks (modules, value data_types, etc.). So, you can't assume that, for example, a pipeline you create
will work the same way (or at all) in a different environment. *kiara* will always be able to tell you all the details
of this environment, though, and it will attach those details to things like data, so there is always a record of
how something was created, and in which environment.
"""
_instances: Dict[str, "Kiara"] = {}
_instance_kiara_config = KiaraConfig()
@classmethod
def instance(
cls,
context_name: Union[str, None] = None,
runtime_config: Union[None, Mapping[str, Any], KiaraRuntimeConfig] = None,
) -> "Kiara":
"""The default *kiara* context. In most cases, it's recommended you create and manage your own, though."""
# TODO: make this thread-safe
if context_name is None:
_context_name = os.environ.get("KIARA_CONTEXT", None)
else:
_context_name = context_name
if not _context_name:
_context_name = cls._instance_kiara_config.default_context
if _context_name in cls._instances.keys():
instance = cls._instances[_context_name]
else:
instance = cls._instance_kiara_config.create_context(context=context_name)
cls._instances[_context_name] = instance
if runtime_config:
if isinstance(runtime_config, KiaraRuntimeConfig):
runtime_config = runtime_config.dict()
instance.update_runtime_config(**runtime_config)
return instance
def __init__(
self,
config: Union[KiaraContextConfig, None] = None,
runtime_config: Union[KiaraRuntimeConfig, None] = None,
):
kc: Union[KiaraConfig, None] = None
if not config:
kc = KiaraConfig()
config = kc.get_context_config()
if not runtime_config:
if kc is None:
kc = KiaraConfig()
runtime_config = kc.runtime_config
self._id: uuid.UUID = ID_REGISTRY.generate(
id=uuid.UUID(config.context_id), obj=self
)
ID_REGISTRY.update_metadata(self._id, kiara_id=self._id)
self._config: KiaraContextConfig = config
self._runtime_config: KiaraRuntimeConfig = runtime_config
self._event_registry: EventRegistry = EventRegistry(kiara=self)
self._type_registry: TypeRegistry = TypeRegistry(self)
self._data_registry: DataRegistry = DataRegistry(kiara=self)
self._job_registry: JobRegistry = JobRegistry(kiara=self)
self._module_registry: ModuleRegistry = ModuleRegistry(kiara=self)
self._operation_registry: OperationRegistry = OperationRegistry(kiara=self)
self._kiara_model_registry: ModelRegistry = ModelRegistry.instance()
self._alias_registry: AliasRegistry = AliasRegistry(kiara=self)
self._destiny_registry: DestinyRegistry = DestinyRegistry(kiara=self)
self._workflow_registry: WorkflowRegistry = WorkflowRegistry(kiara=self)
self._env_mgmt: Union[EnvironmentRegistry, None] = None
metadata_augmenter = CreateMetadataDestinies(kiara=self)
self._event_registry.add_listener(
metadata_augmenter, *metadata_augmenter.supported_event_types()
)
self._context_info: Union[KiaraContextInfo, None] = None
# initialize stores
self._archive_types = find_all_archive_types()
self._archives: Dict[str, KiaraArchive] = {}
for archive_alias, archive in self._config.archives.items():
archive_cls = self._archive_types.get(archive.archive_type, None)
if archive_cls is None:
raise Exception(
f"Can't create context: no archive type '{archive.archive_type}' available. Available types: {', '.join(self._archive_types.keys())}"
)
config_cls = archive_cls._config_cls
archive_config = config_cls(**archive.config)
archive_obj = archive_cls(archive_id=archive.archive_uuid, config=archive_config) # type: ignore
for supported_type in archive_obj.supported_item_types():
if supported_type == "data":
self.data_registry.register_data_archive(
archive_obj, alias=archive_alias # type: ignore
)
if supported_type == "job_record":
self.job_registry.register_job_archive(archive_obj, alias=archive_alias) # type: ignore
if supported_type == "alias":
self.alias_registry.register_archive(archive_obj, alias=archive_alias) # type: ignore
if supported_type == "destiny":
self.destiny_registry.register_destiny_archive(archive_obj, alias=archive_alias) # type: ignore
if supported_type == "workflow":
self.workflow_registry.register_archive(archive_obj, alias=archive_alias) # type: ignore
def _run_alembic_migrations(self):
script_location = os.path.abspath(KIARA_DB_MIGRATIONS_FOLDER)
dsn = self._config.db_url
log_message("running migration script", script=script_location, db_url=dsn)
from alembic.config import Config
alembic_cfg = Config(KIARA_DB_MIGRATIONS_CONFIG)
alembic_cfg.set_main_option("script_location", script_location)
alembic_cfg.set_main_option("sqlalchemy.url", dsn)
command.upgrade(alembic_cfg, "head")
@property
def id(self) -> uuid.UUID:
return self._id
@property
def context_config(self) -> KiaraContextConfig:
return self._config
@property
def runtime_config(self) -> KiaraRuntimeConfig:
return self._runtime_config
def update_runtime_config(self, **settings) -> KiaraRuntimeConfig:
for k, v in settings.items():
setattr(self.runtime_config, k, v)
return self.runtime_config
@property
def context_info(self) -> "KiaraContextInfo":
if self._context_info is None:
self._context_info = KiaraContextInfo.create_from_kiara_instance(kiara=self)
return self._context_info
# ===================================================================================================
# registry accessors
@property
def environment_registry(self) -> EnvironmentRegistry:
if self._env_mgmt is not None:
return self._env_mgmt
self._env_mgmt = EnvironmentRegistry.instance()
return self._env_mgmt
@property
def type_registry(self) -> TypeRegistry:
return self._type_registry
@property
def module_registry(self) -> ModuleRegistry:
return self._module_registry
@property
def kiara_model_registry(self) -> ModelRegistry:
return self._kiara_model_registry
@property
def alias_registry(self) -> AliasRegistry:
return self._alias_registry
@property
def destiny_registry(self) -> DestinyRegistry:
return self._destiny_registry
@property
def job_registry(self) -> JobRegistry:
return self._job_registry
@property
def operation_registry(self) -> OperationRegistry:
op_registry = self._operation_registry
return op_registry
@property
def data_registry(self) -> DataRegistry:
return self._data_registry
@property
def workflow_registry(self) -> WorkflowRegistry:
return self._workflow_registry
@property
def event_registry(self) -> EventRegistry:
return self._event_registry
# ===================================================================================================
# context specific types & instances
@property
def current_environments(self) -> Mapping[str, RuntimeEnvironment]:
return self.environment_registry.environments
@property
def data_type_classes(self) -> Mapping[str, Type[DataType]]:
return self.type_registry.data_type_classes
@property
def data_type_names(self) -> List[str]:
return self.type_registry.data_type_names
@property
def module_type_classes(self) -> Mapping[str, Type["KiaraModule"]]:
return self._module_registry.module_types
@property
def module_type_names(self) -> Iterable[str]:
return self._module_registry.get_module_type_names()
# ===================================================================================================
# kiara session API methods
def create_manifest(
self, module_or_operation: str, config: Union[Mapping[str, Any], None] = None
) -> Manifest:
if config is None:
config = {}
if module_or_operation in self.module_type_names:
manifest: Manifest = Manifest(
module_type=module_or_operation, module_config=config
)
elif module_or_operation in self.operation_registry.operation_ids:
if config:
raise Exception(
f"Specified run target '{module_or_operation}' is an operation, additional module configuration is not allowed (yet)."
)
manifest = self.operation_registry.get_operation(module_or_operation)
elif os.path.isfile(module_or_operation):
raise NotImplementedError()
else:
raise Exception(
f"Can't assemble operation, invalid operation/module name: {module_or_operation}. Must be registered module or operation name, or file."
)
return manifest
def create_module(self, manifest: Union[Manifest, str]) -> "KiaraModule":
"""Create a [KiaraModule][kiara.module.KiaraModule] object from a module configuration.
Arguments:
manifest: the module configuration
"""
return self._module_registry.create_module(manifest=manifest)
def queue(
self, manifest: Manifest, inputs: Mapping[str, Any], wait: bool = False
) -> uuid.UUID:
"""Queue a job with the specified manifest and inputs.
Arguments:
manifest: the job manifest
inputs: the job inputs
wait: whether to wait for the job to be finished before returning
Returns:
the job id that can be used to look up job status & results
"""
return self.job_registry.execute(manifest=manifest, inputs=inputs, wait=wait)
def process(self, manifest: Manifest, inputs: Mapping[str, Any]) -> ValueMap:
"""Queue a job with the specified manifest and inputs.
Arguments:
manifest: the job manifest
inputs: the job inputs
wait: whether to wait for the job to be finished before returning
Returns
"""
return self.job_registry.execute_and_retrieve(manifest=manifest, inputs=inputs)
def save_values(
self, values: ValueMap, alias_map: Mapping[str, Iterable[str]]
) -> StoreValuesResult:
_values = {}
for field_name in values.field_names:
value = values.get_value_obj(field_name)
_values[field_name] = value
self.data_registry.store_value(value=value)
stored = {}
for field_name, field_aliases in alias_map.items():
value = _values[field_name]
try:
if field_aliases:
self.alias_registry.register_aliases(value.value_id, *field_aliases)
stored[field_name] = StoreValueResult.construct(
value=value, aliases=sorted(field_aliases), error=None
)
except Exception as e:
log_exception(e)
stored[field_name] = StoreValueResult.construct(
value=value, aliases=sorted(field_aliases), error=str(e)
)
return StoreValuesResult.construct(__root__=stored)
def create_context_summary(self) -> ContextSummary:
return ContextSummary.create_from_context(kiara=self)
def get_all_archives(self) -> Dict[KiaraArchive, Set[str]]:
result: Dict[KiaraArchive, Set[str]] = {}
archive: KiaraArchive
for alias, archive in self.data_registry.data_archives.items():
result.setdefault(archive, set()).add(alias)
for alias, archive in self.alias_registry.alias_archives.items():
result.setdefault(archive, set()).add(alias)
for alias, archive in self.destiny_registry.destiny_archives.items():
result.setdefault(archive, set()).add(alias)
for alias, archive in self.job_registry.job_archives.items():
result.setdefault(archive, set()).add(alias)
for alias, archive in self.workflow_registry.workflow_archives.items():
result.setdefault(archive, set()).add(alias)
return result
alias_registry: AliasRegistry
property
readonly
¶context_config: KiaraContextConfig
property
readonly
¶context_info: KiaraContextInfo
property
readonly
¶current_environments: Mapping[str, kiara.models.runtime_environment.RuntimeEnvironment]
property
readonly
¶data_registry: DataRegistry
property
readonly
¶data_type_classes: Mapping[str, Type[kiara.data_types.DataType]]
property
readonly
¶data_type_names: List[str]
property
readonly
¶destiny_registry: DestinyRegistry
property
readonly
¶environment_registry: EnvironmentRegistry
property
readonly
¶event_registry: EventRegistry
property
readonly
¶id: UUID
property
readonly
¶job_registry: JobRegistry
property
readonly
¶kiara_model_registry: ModelRegistry
property
readonly
¶module_registry: ModuleRegistry
property
readonly
¶module_type_classes: Mapping[str, Type[KiaraModule]]
property
readonly
¶module_type_names: Iterable[str]
property
readonly
¶operation_registry: OperationRegistry
property
readonly
¶runtime_config: KiaraRuntimeConfig
property
readonly
¶type_registry: TypeRegistry
property
readonly
¶workflow_registry: WorkflowRegistry
property
readonly
¶Methods¶
create_context_summary(self)
¶Source code in kiara/context/__init__.py
def create_context_summary(self) -> ContextSummary:
return ContextSummary.create_from_context(kiara=self)
create_manifest(self, module_or_operation, config=None)
¶Source code in kiara/context/__init__.py
def create_manifest(
self, module_or_operation: str, config: Union[Mapping[str, Any], None] = None
) -> Manifest:
if config is None:
config = {}
if module_or_operation in self.module_type_names:
manifest: Manifest = Manifest(
module_type=module_or_operation, module_config=config
)
elif module_or_operation in self.operation_registry.operation_ids:
if config:
raise Exception(
f"Specified run target '{module_or_operation}' is an operation, additional module configuration is not allowed (yet)."
)
manifest = self.operation_registry.get_operation(module_or_operation)
elif os.path.isfile(module_or_operation):
raise NotImplementedError()
else:
raise Exception(
f"Can't assemble operation, invalid operation/module name: {module_or_operation}. Must be registered module or operation name, or file."
)
return manifest
create_module(self, manifest)
¶Create a [KiaraModule][kiara.module.KiaraModule] object from a module configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
manifest |
Union[kiara.models.module.manifest.Manifest, str] |
the module configuration |
required |
Source code in kiara/context/__init__.py
def create_module(self, manifest: Union[Manifest, str]) -> "KiaraModule":
"""Create a [KiaraModule][kiara.module.KiaraModule] object from a module configuration.
Arguments:
manifest: the module configuration
"""
return self._module_registry.create_module(manifest=manifest)
get_all_archives(self)
¶Source code in kiara/context/__init__.py
def get_all_archives(self) -> Dict[KiaraArchive, Set[str]]:
result: Dict[KiaraArchive, Set[str]] = {}
archive: KiaraArchive
for alias, archive in self.data_registry.data_archives.items():
result.setdefault(archive, set()).add(alias)
for alias, archive in self.alias_registry.alias_archives.items():
result.setdefault(archive, set()).add(alias)
for alias, archive in self.destiny_registry.destiny_archives.items():
result.setdefault(archive, set()).add(alias)
for alias, archive in self.job_registry.job_archives.items():
result.setdefault(archive, set()).add(alias)
for alias, archive in self.workflow_registry.workflow_archives.items():
result.setdefault(archive, set()).add(alias)
return result
instance(context_name=None, runtime_config=None)
classmethod
¶The default kiara context. In most cases, it's recommended you create and manage your own, though.
Source code in kiara/context/__init__.py
@classmethod
def instance(
cls,
context_name: Union[str, None] = None,
runtime_config: Union[None, Mapping[str, Any], KiaraRuntimeConfig] = None,
) -> "Kiara":
"""The default *kiara* context. In most cases, it's recommended you create and manage your own, though."""
# TODO: make this thread-safe
if context_name is None:
_context_name = os.environ.get("KIARA_CONTEXT", None)
else:
_context_name = context_name
if not _context_name:
_context_name = cls._instance_kiara_config.default_context
if _context_name in cls._instances.keys():
instance = cls._instances[_context_name]
else:
instance = cls._instance_kiara_config.create_context(context=context_name)
cls._instances[_context_name] = instance
if runtime_config:
if isinstance(runtime_config, KiaraRuntimeConfig):
runtime_config = runtime_config.dict()
instance.update_runtime_config(**runtime_config)
return instance
process(self, manifest, inputs)
¶Queue a job with the specified manifest and inputs.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
manifest |
Manifest |
the job manifest |
required |
inputs |
Mapping[str, Any] |
the job inputs |
required |
wait |
whether to wait for the job to be finished before returning |
required |
Returns
Source code in kiara/context/__init__.py
def process(self, manifest: Manifest, inputs: Mapping[str, Any]) -> ValueMap:
"""Queue a job with the specified manifest and inputs.
Arguments:
manifest: the job manifest
inputs: the job inputs
wait: whether to wait for the job to be finished before returning
Returns
"""
return self.job_registry.execute_and_retrieve(manifest=manifest, inputs=inputs)
queue(self, manifest, inputs, wait=False)
¶Queue a job with the specified manifest and inputs.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
manifest |
Manifest |
the job manifest |
required |
inputs |
Mapping[str, Any] |
the job inputs |
required |
wait |
bool |
whether to wait for the job to be finished before returning |
False |
Returns:
| Type | Description |
|---|---|
UUID |
the job id that can be used to look up job status & results |
Source code in kiara/context/__init__.py
def queue(
self, manifest: Manifest, inputs: Mapping[str, Any], wait: bool = False
) -> uuid.UUID:
"""Queue a job with the specified manifest and inputs.
Arguments:
manifest: the job manifest
inputs: the job inputs
wait: whether to wait for the job to be finished before returning
Returns:
the job id that can be used to look up job status & results
"""
return self.job_registry.execute(manifest=manifest, inputs=inputs, wait=wait)
save_values(self, values, alias_map)
¶Source code in kiara/context/__init__.py
def save_values(
self, values: ValueMap, alias_map: Mapping[str, Iterable[str]]
) -> StoreValuesResult:
_values = {}
for field_name in values.field_names:
value = values.get_value_obj(field_name)
_values[field_name] = value
self.data_registry.store_value(value=value)
stored = {}
for field_name, field_aliases in alias_map.items():
value = _values[field_name]
try:
if field_aliases:
self.alias_registry.register_aliases(value.value_id, *field_aliases)
stored[field_name] = StoreValueResult.construct(
value=value, aliases=sorted(field_aliases), error=None
)
except Exception as e:
log_exception(e)
stored[field_name] = StoreValueResult.construct(
value=value, aliases=sorted(field_aliases), error=str(e)
)
return StoreValuesResult.construct(__root__=stored)
update_runtime_config(self, **settings)
¶Source code in kiara/context/__init__.py
def update_runtime_config(self, **settings) -> KiaraRuntimeConfig:
for k, v in settings.items():
setattr(self.runtime_config, k, v)
return self.runtime_config
KiaraContextInfo (KiaraModel)
pydantic-model
¶
Source code in kiara/context/__init__.py
class KiaraContextInfo(KiaraModel):
@classmethod
def create_from_kiara_instance(
cls, kiara: "Kiara", package_filter: Union[str, None] = None
):
data_types = kiara.type_registry.get_context_metadata(
only_for_package=package_filter
)
modules = kiara.module_registry.get_context_metadata(
only_for_package=package_filter
)
operation_types = kiara.operation_registry.get_context_metadata(
only_for_package=package_filter
)
operations = filter_operations(
kiara=kiara, pkg_name=package_filter, **kiara.operation_registry.operations
)
model_registry = kiara.kiara_model_registry
if package_filter:
kiara_models = model_registry.get_models_for_package(
package_name=package_filter
)
else:
kiara_models = model_registry.all_models
# metadata_types = find_metadata_models(only_for_package=package_filter)
return KiaraContextInfo.construct(
kiara_id=kiara.id,
package_filter=package_filter,
data_types=data_types,
module_types=modules,
kiara_model_types=kiara_models,
# metadata_types=metadata_types,
operation_types=operation_types,
operations=operations,
)
kiara_id: uuid.UUID = Field(description="The id of the kiara context.")
package_filter: Union[str, None] = Field(
description="Whether this context is filtered to only include information included in a specific Python package."
)
data_types: DataTypeClassesInfo = Field(description="The included data types.")
module_types: ModuleTypesInfo = Field(
description="The included kiara module types."
)
kiara_model_types: KiaraModelClassesInfo = Field(
description="The included model classes."
)
# metadata_types: MetadataTypeClassesInfo = Field(
# description="The included value metadata types."
# )
operation_types: OperationTypeClassesInfo = Field(
description="The included operation types."
)
operations: OperationGroupInfo = Field(description="The included operations.")
def _retrieve_id(self) -> str:
if not self.package_filter:
return str(self.kiara_id)
else:
return f"{self.kiara_id}.package_{self.package_filter}"
def _retrieve_data_to_hash(self) -> Any:
return {"kiara_id": self.kiara_id, "package": self.package_filter}
def get_info(self, item_type: str, item_id: str) -> ItemInfo:
if "data_type" == item_type or "data_types" == item_type:
group_info: InfoItemGroup = self.data_types
elif "module" in item_type:
group_info = self.module_types
# elif "metadata" in item_type:
# group_info = self.metadata_types
elif "operation_type" in item_type or "operation_types" in item_type:
group_info = self.operation_types
elif "operation" in item_type:
group_info = self.operations
elif "kiara_model" in item_type:
group_info = self.kiara_model_types
else:
item_types = [
"data_type",
"module_type",
"kiara_model_type",
"operation_type",
"operation",
]
raise Exception(
f"Can't determine item type '{item_type}', use one of: {', '.join(item_types)}"
)
return group_info[item_id]
def get_all_info(self, skip_empty_types: bool = True) -> Dict[str, InfoItemGroup]:
result: Dict[str, InfoItemGroup] = {}
if self.data_types or not skip_empty_types:
result["data_types"] = self.data_types
if self.module_types or not skip_empty_types:
result["module_types"] = self.module_types
if self.kiara_model_types or not skip_empty_types:
result["kiara_model_types"] = self.kiara_model_types
# if self.metadata_types or not skip_empty_types:
# result["metadata_types"] = self.metadata_types
if self.operation_types or not skip_empty_types:
result["operation_types"] = self.operation_types
if self.operations or not skip_empty_types:
result["operations"] = self.operations
return result
Attributes¶
data_types: DataTypeClassesInfo
pydantic-field
required
¶The included data types.
kiara_id: UUID
pydantic-field
required
¶The id of the kiara context.
kiara_model_types: KiaraModelClassesInfo
pydantic-field
required
¶The included model classes.
module_types: ModuleTypesInfo
pydantic-field
required
¶The included kiara module types.
operation_types: OperationTypeClassesInfo
pydantic-field
required
¶The included operation types.
operations: OperationGroupInfo
pydantic-field
required
¶The included operations.
package_filter: str
pydantic-field
¶Whether this context is filtered to only include information included in a specific Python package.
create_from_kiara_instance(kiara, package_filter=None)
classmethod
¶Source code in kiara/context/__init__.py
@classmethod
def create_from_kiara_instance(
cls, kiara: "Kiara", package_filter: Union[str, None] = None
):
data_types = kiara.type_registry.get_context_metadata(
only_for_package=package_filter
)
modules = kiara.module_registry.get_context_metadata(
only_for_package=package_filter
)
operation_types = kiara.operation_registry.get_context_metadata(
only_for_package=package_filter
)
operations = filter_operations(
kiara=kiara, pkg_name=package_filter, **kiara.operation_registry.operations
)
model_registry = kiara.kiara_model_registry
if package_filter:
kiara_models = model_registry.get_models_for_package(
package_name=package_filter
)
else:
kiara_models = model_registry.all_models
# metadata_types = find_metadata_models(only_for_package=package_filter)
return KiaraContextInfo.construct(
kiara_id=kiara.id,
package_filter=package_filter,
data_types=data_types,
module_types=modules,
kiara_model_types=kiara_models,
# metadata_types=metadata_types,
operation_types=operation_types,
operations=operations,
)
get_all_info(self, skip_empty_types=True)
¶Source code in kiara/context/__init__.py
def get_all_info(self, skip_empty_types: bool = True) -> Dict[str, InfoItemGroup]:
result: Dict[str, InfoItemGroup] = {}
if self.data_types or not skip_empty_types:
result["data_types"] = self.data_types
if self.module_types or not skip_empty_types:
result["module_types"] = self.module_types
if self.kiara_model_types or not skip_empty_types:
result["kiara_model_types"] = self.kiara_model_types
# if self.metadata_types or not skip_empty_types:
# result["metadata_types"] = self.metadata_types
if self.operation_types or not skip_empty_types:
result["operation_types"] = self.operation_types
if self.operations or not skip_empty_types:
result["operations"] = self.operations
return result
get_info(self, item_type, item_id)
¶Source code in kiara/context/__init__.py
def get_info(self, item_type: str, item_id: str) -> ItemInfo:
if "data_type" == item_type or "data_types" == item_type:
group_info: InfoItemGroup = self.data_types
elif "module" in item_type:
group_info = self.module_types
# elif "metadata" in item_type:
# group_info = self.metadata_types
elif "operation_type" in item_type or "operation_types" in item_type:
group_info = self.operation_types
elif "operation" in item_type:
group_info = self.operations
elif "kiara_model" in item_type:
group_info = self.kiara_model_types
else:
item_types = [
"data_type",
"module_type",
"kiara_model_type",
"operation_type",
"operation",
]
raise Exception(
f"Can't determine item type '{item_type}', use one of: {', '.join(item_types)}"
)
return group_info[item_id]
Functions¶
explain(item, kiara=None)
¶
Pretty print information about an item on the terminal.
Source code in kiara/context/__init__.py
def explain(item: Any, kiara: Union[None, "Kiara"] = None):
"""Pretty print information about an item on the terminal."""
if isinstance(item, type):
from kiara.modules import KiaraModule
if issubclass(item, KiaraModule):
if kiara is None:
kiara = Kiara.instance()
item = ModuleTypeInfo.create_from_type_class(type_cls=item, kiara=kiara)
console = get_console()
console.print(item)
Modules¶
config
¶
logger
¶yaml
¶Classes¶
JobCacheStrategy (Enum)
¶
KiaraArchiveConfig (BaseModel)
pydantic-model
¶Source code in kiara/context/config.py
class KiaraArchiveConfig(BaseModel):
archive_id: str = Field(description="The unique archive id.")
archive_type: str = Field(description="The archive type.")
config: Mapping[str, Any] = Field(
description="Archive type specific config.", default_factory=dict
)
@property
def archive_uuid(self) -> uuid.UUID:
return uuid.UUID(self.archive_id)
KiaraConfig (BaseSettings)
pydantic-model
¶Source code in kiara/context/config.py
class KiaraConfig(BaseSettings):
class Config:
env_prefix = "kiara_"
extra = Extra.forbid
use_enum_values = True
# @classmethod
# def customise_sources(
# cls,
# init_settings,
# env_settings,
# file_secret_settings,
# ):
# return (init_settings, env_settings, config_file_settings_source)
@classmethod
def create_in_folder(cls, path: Union[Path, str]) -> "KiaraConfig":
if isinstance(path, str):
path = Path(path)
path = path.absolute()
if path.exists():
raise Exception(
f"Can't create new kiara config, path exists: {path.as_posix()}"
)
config = KiaraConfig(base_data_path=path)
config_file = path / KIARA_CONFIG_FILE_NAME
config.save(config_file)
return config
@classmethod
def load_from_file(cls, path: Union[Path, None] = None) -> "KiaraConfig":
if path is None:
path = Path(KIARA_MAIN_CONFIG_FILE)
if not path.exists():
raise Exception(
f"Can't load kiara config, path does not exist: {path.as_posix()}"
)
if path.is_dir():
path = path / KIARA_CONFIG_FILE_NAME
if not path.exists():
raise Exception(
f"Can't load kiara config, path does not exist: {path.as_posix()}"
)
with path.open("rt") as f:
data = yaml.load(f)
config = KiaraConfig(**data)
config._config_path = path
return config
context_search_paths: List[str] = Field(
description="The base path to look for contexts in.",
default=[KIARA_MAIN_CONTEXTS_PATH],
)
base_data_path: str = Field(
description="The base path to use for all data (unless otherwise specified.",
default=kiara_app_dirs.user_data_dir,
)
stores_base_path: str = Field(
description="The base path for the stores of this context."
)
default_context: str = Field(
description="The name of the default context to use if none is provided.",
default=DEFAULT_CONTEXT_NAME,
)
auto_generate_contexts: bool = Field(
description="Whether to auto-generate requested contexts if they don't exist yet.",
default=True,
)
runtime_config: KiaraRuntimeConfig = Field(
description="The kiara runtime config.", default_factory=KiaraRuntimeConfig
)
_contexts: Dict[uuid.UUID, "Kiara"] = PrivateAttr(default_factory=dict)
_available_context_files: Dict[str, Path] = PrivateAttr(default=None)
_context_data: Dict[str, KiaraContextConfig] = PrivateAttr(default_factory=dict)
_config_path: Union[Path, None] = PrivateAttr(default=None)
@validator("context_search_paths")
def validate_context_search_paths(cls, v):
if not v or not v[0]:
v = [KIARA_MAIN_CONTEXTS_PATH]
return v
@root_validator(pre=True)
def _set_paths(cls, values: Any):
base_path = values.get("base_data_path", None)
if not base_path:
base_path = os.path.abspath(kiara_app_dirs.user_data_dir)
values["base_data_path"] = base_path
elif isinstance(base_path, Path):
base_path = base_path.absolute().as_posix()
values["base_data_path"] = base_path
stores_base_path = values.get("stores_base_path", None)
if not stores_base_path:
stores_base_path = os.path.join(base_path, "stores")
values["stores_base_path"] = stores_base_path
context_search_paths = values.get("context_search_paths")
if not context_search_paths:
context_search_paths = [os.path.join(base_path, "contexts")]
values["context_search_paths"] = context_search_paths
return values
@property
def available_context_names(self) -> Iterable[str]:
if self._available_context_files is not None:
return self._available_context_files.keys()
result = {}
for search_path in self.context_search_paths:
sp = Path(search_path)
for path in sp.rglob("*.yaml"):
rel_path = path.relative_to(sp)
alias = rel_path.as_posix()[0:-5]
alias = alias.replace(os.sep, ".")
result[alias] = path
self._available_context_files = result
return self._available_context_files.keys()
@property
def context_configs(self) -> Mapping[str, KiaraContextConfig]:
return {a: self.get_context_config(a) for a in self.available_context_names}
def get_context_config(
self,
context_name: Union[str, None] = None,
auto_generate: Union[bool, None] = None,
) -> KiaraContextConfig:
if auto_generate is None:
auto_generate = self.auto_generate_contexts
if context_name is None:
context_name = self.default_context
if context_name not in self.available_context_names:
if not auto_generate and not context_name == DEFAULT_CONTEXT_NAME:
raise Exception(
f"No kiara context with name '{context_name}' available."
)
else:
return self.create_context_config(context_alias=context_name)
if context_name in self._context_data.keys():
return self._context_data[context_name]
context_file = self._available_context_files[context_name]
context_data = get_data_from_file(context_file, content_type="yaml")
changed = False
if "extra_pipeline_folders" in context_data.keys():
epf = context_data.pop("extra_pipeline_folders")
context_data.setdefault("extra_pipelines", []).extend(epf)
changed = True
context = KiaraContextConfig(**context_data)
if not changed:
changed = self._validate_context(context_config=context)
if changed:
logger.debug(
"write.context_file",
context_config_file=context_file.as_posix(),
context_name=context_name,
reason="context changed after validation",
)
context_file.parent.mkdir(parents=True, exist_ok=True)
with open(context_file, "wt") as f:
yaml.dump(context.dict(), f)
context._context_config_path = context_file
self._context_data[context_name] = context
return context
def _validate_context(self, context_config: KiaraContextConfig) -> bool:
env_registry = EnvironmentRegistry.instance()
available_archives = env_registry.environments["kiara_types"].archive_types
changed = False
if DEFAULT_DATA_STORE_MARKER not in context_config.archives.keys():
data_store_type = "filesystem_data_store"
assert data_store_type in available_archives.item_infos.keys()
data_store_id = ID_REGISTRY.generate(comment="default data store id")
data_archive_config = {
"archive_path": os.path.abspath(
os.path.join(
self.stores_base_path, data_store_type, str(data_store_id)
)
)
}
data_store = KiaraArchiveConfig.construct(
archive_id=str(data_store_id),
archive_type=data_store_type,
config=data_archive_config,
)
context_config.archives[DEFAULT_DATA_STORE_MARKER] = data_store
changed = True
if DEFAULT_JOB_STORE_MARKER not in context_config.archives.keys():
job_store_type = "filesystem_job_store"
assert job_store_type in available_archives.item_infos.keys()
job_store_id = ID_REGISTRY.generate(comment="default job store id")
job_archive_config = {
"archive_path": os.path.abspath(
os.path.join(
self.stores_base_path, job_store_type, str(job_store_id)
)
)
}
job_store = KiaraArchiveConfig.construct(
archive_id=str(job_store_id),
archive_type=job_store_type,
config=job_archive_config,
)
context_config.archives[DEFAULT_JOB_STORE_MARKER] = job_store
changed = True
if DEFAULT_ALIAS_STORE_MARKER not in context_config.archives.keys():
alias_store_type = "filesystem_alias_store"
assert alias_store_type in available_archives.item_infos.keys()
alias_store_id = ID_REGISTRY.generate(comment="default alias store id")
alias_store_config = {
"archive_path": os.path.abspath(
os.path.join(
self.stores_base_path, alias_store_type, str(alias_store_id)
)
)
}
alias_store = KiaraArchiveConfig.construct(
archive_id=str(alias_store_id),
archive_type=alias_store_type,
config=alias_store_config,
)
context_config.archives[DEFAULT_ALIAS_STORE_MARKER] = alias_store
changed = True
if DEFAULT_WORKFLOW_STORE_MARKER not in context_config.archives.keys():
workflow_store_type = "filesystem_workflow_store"
assert workflow_store_type in available_archives.item_infos.keys()
workflow_store_id = ID_REGISTRY.generate(
comment="default workflow store id"
)
workflow_store_config = {
"archive_path": os.path.abspath(
os.path.join(
self.stores_base_path,
workflow_store_type,
str(workflow_store_id),
)
)
}
workflow_store = KiaraArchiveConfig.construct(
archive_id=str(workflow_store_id),
archive_type=workflow_store_type,
config=workflow_store_config,
)
context_config.archives[DEFAULT_WORKFLOW_STORE_MARKER] = workflow_store
changed = True
if METADATA_DESTINY_STORE_MARKER not in context_config.archives.keys():
destiny_store_type = "filesystem_destiny_store"
assert destiny_store_type in available_archives.item_infos.keys()
destiny_store_id = ID_REGISTRY.generate(comment="default destiny store id")
destiny_store_config = {
"archive_path": os.path.abspath(
os.path.join(
self.stores_base_path, destiny_store_type, str(destiny_store_id)
)
)
}
destiny_store = KiaraArchiveConfig.construct(
archive_id=str(destiny_store_id),
archive_type=destiny_store_type,
config=destiny_store_config,
)
context_config.archives[METADATA_DESTINY_STORE_MARKER] = destiny_store
changed = True
return changed
def create_context_config(
self, context_alias: Union[str, None] = None
) -> KiaraContextConfig:
if not context_alias:
context_alias = DEFAULT_CONTEXT_NAME
if context_alias in self.available_context_names:
raise Exception(
f"Can't create kiara context '{context_alias}': context with that alias already registered."
)
if os.path.sep in context_alias:
raise Exception(
f"Can't create context with alias '{context_alias}': no special characters allowed."
)
context_file = (
Path(os.path.join(self.context_search_paths[0])) / f"{context_alias}.yaml"
)
archives: Dict[str, KiaraArchiveConfig] = {}
# create_default_archives(kiara_config=self)
context_id = ID_REGISTRY.generate(
obj_type=KiaraContextConfig, comment=f"new kiara context '{context_alias}'"
)
context_config = KiaraContextConfig(
context_id=str(context_id), archives=archives, extra_pipelines=[]
)
self._validate_context(context_config=context_config)
context_file.parent.mkdir(parents=True, exist_ok=True)
with open(context_file, "wt") as f:
yaml.dump(context_config.dict(), f)
context_config._context_config_path = context_file
self._available_context_files[context_alias] = context_file
self._context_data[context_alias] = context_config
return context_config
def create_context(
self,
context: Union[None, str, uuid.UUID, Path] = None,
extra_pipelines: Union[None, str, Iterable[str]] = None,
) -> "Kiara":
if not context:
context = DEFAULT_CONTEXT_NAME
else:
try:
context = uuid.UUID(context) # type: ignore
except Exception:
pass
if isinstance(context, str) and os.path.exists(context):
context = Path(context)
if isinstance(context, Path):
with context.open("rt") as f:
data = yaml.load(f)
context_config = KiaraContextConfig(**data)
elif isinstance(context, str):
context_config = self.get_context_config(context_name=context)
elif isinstance(context, uuid.UUID):
context_config = self.find_context_config(context_id=context)
else:
raise Exception(
f"Can't retrieve context, invalid context config type '{type(context)}'."
)
assert context_config.context_id not in self._contexts.keys()
if extra_pipelines:
if isinstance(extra_pipelines, str):
extra_pipelines = [extra_pipelines]
context_config.add_pipelines(*extra_pipelines)
from kiara.context import Kiara
kiara = Kiara(config=context_config, runtime_config=self.runtime_config)
assert kiara.id == uuid.UUID(context_config.context_id)
self._contexts[kiara.id] = kiara
return kiara
def find_context_config(self, context_id: uuid.UUID) -> KiaraContextConfig:
raise NotImplementedError()
def save(self, path: Union[Path, None] = None):
if path is None:
path = Path(KIARA_MAIN_CONFIG_FILE)
if path.exists():
raise Exception(
f"Can't save config file, path already exists: {path.as_posix()}"
)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("wt") as f:
yaml.dump(
self.dict(
exclude={
"context",
"auto_generate_contexts",
"stores_base_path",
"context_search_paths",
"default_context",
"runtime_config",
}
),
f,
)
self._config_path = path
def delete(
self, context_name: Union[str, None] = None, dry_run: bool = True
) -> "ContextSummary":
if context_name is None:
context_name = self.default_context
from kiara.context import Kiara
from kiara.models.context import ContextSummary
context_config = self.get_context_config(
context_name=context_name, auto_generate=False
)
kiara = Kiara(config=context_config, runtime_config=self.runtime_config)
context_summary = ContextSummary.create_from_context(
kiara=kiara, context_name=context_name
)
if dry_run:
return context_summary
for archive in kiara.get_all_archives().keys():
archive.delete_archive(archive_id=archive.archive_id)
if context_config._context_config_path is not None:
os.unlink(context_config._context_config_path)
return context_summary
def create_renderable(self, **render_config: Any):
from kiara.utils.output import create_recursive_table_from_model_object
return create_recursive_table_from_model_object(
self, render_config=render_config
)
auto_generate_contexts: bool
pydantic-field
¶Whether to auto-generate requested contexts if they don't exist yet.
available_context_names: Iterable[str]
property
readonly
¶base_data_path: str
pydantic-field
¶The base path to use for all data (unless otherwise specified.
context_configs: Mapping[str, kiara.context.config.KiaraContextConfig]
property
readonly
¶context_search_paths: List[str]
pydantic-field
¶The base path to look for contexts in.
default_context: str
pydantic-field
¶The name of the default context to use if none is provided.
runtime_config: KiaraRuntimeConfig
pydantic-field
¶The kiara runtime config.
stores_base_path: str
pydantic-field
required
¶The base path for the stores of this context.
Config
¶Source code in kiara/context/config.py
class Config:
env_prefix = "kiara_"
extra = Extra.forbid
use_enum_values = True
# @classmethod
# def customise_sources(
# cls,
# init_settings,
# env_settings,
# file_secret_settings,
# ):
# return (init_settings, env_settings, config_file_settings_source)
create_context(self, context=None, extra_pipelines=None)
¶Source code in kiara/context/config.py
def create_context(
self,
context: Union[None, str, uuid.UUID, Path] = None,
extra_pipelines: Union[None, str, Iterable[str]] = None,
) -> "Kiara":
if not context:
context = DEFAULT_CONTEXT_NAME
else:
try:
context = uuid.UUID(context) # type: ignore
except Exception:
pass
if isinstance(context, str) and os.path.exists(context):
context = Path(context)
if isinstance(context, Path):
with context.open("rt") as f:
data = yaml.load(f)
context_config = KiaraContextConfig(**data)
elif isinstance(context, str):
context_config = self.get_context_config(context_name=context)
elif isinstance(context, uuid.UUID):
context_config = self.find_context_config(context_id=context)
else:
raise Exception(
f"Can't retrieve context, invalid context config type '{type(context)}'."
)
assert context_config.context_id not in self._contexts.keys()
if extra_pipelines:
if isinstance(extra_pipelines, str):
extra_pipelines = [extra_pipelines]
context_config.add_pipelines(*extra_pipelines)
from kiara.context import Kiara
kiara = Kiara(config=context_config, runtime_config=self.runtime_config)
assert kiara.id == uuid.UUID(context_config.context_id)
self._contexts[kiara.id] = kiara
return kiara
create_context_config(self, context_alias=None)
¶Source code in kiara/context/config.py
def create_context_config(
self, context_alias: Union[str, None] = None
) -> KiaraContextConfig:
if not context_alias:
context_alias = DEFAULT_CONTEXT_NAME
if context_alias in self.available_context_names:
raise Exception(
f"Can't create kiara context '{context_alias}': context with that alias already registered."
)
if os.path.sep in context_alias:
raise Exception(
f"Can't create context with alias '{context_alias}': no special characters allowed."
)
context_file = (
Path(os.path.join(self.context_search_paths[0])) / f"{context_alias}.yaml"
)
archives: Dict[str, KiaraArchiveConfig] = {}
# create_default_archives(kiara_config=self)
context_id = ID_REGISTRY.generate(
obj_type=KiaraContextConfig, comment=f"new kiara context '{context_alias}'"
)
context_config = KiaraContextConfig(
context_id=str(context_id), archives=archives, extra_pipelines=[]
)
self._validate_context(context_config=context_config)
context_file.parent.mkdir(parents=True, exist_ok=True)
with open(context_file, "wt") as f:
yaml.dump(context_config.dict(), f)
context_config._context_config_path = context_file
self._available_context_files[context_alias] = context_file
self._context_data[context_alias] = context_config
return context_config
create_in_folder(path)
classmethod
¶Source code in kiara/context/config.py
@classmethod
def create_in_folder(cls, path: Union[Path, str]) -> "KiaraConfig":
if isinstance(path, str):
path = Path(path)
path = path.absolute()
if path.exists():
raise Exception(
f"Can't create new kiara config, path exists: {path.as_posix()}"
)
config = KiaraConfig(base_data_path=path)
config_file = path / KIARA_CONFIG_FILE_NAME
config.save(config_file)
return config
create_renderable(self, **render_config)
¶Source code in kiara/context/config.py
def create_renderable(self, **render_config: Any):
from kiara.utils.output import create_recursive_table_from_model_object
return create_recursive_table_from_model_object(
self, render_config=render_config
)
delete(self, context_name=None, dry_run=True)
¶Source code in kiara/context/config.py
def delete(
self, context_name: Union[str, None] = None, dry_run: bool = True
) -> "ContextSummary":
if context_name is None:
context_name = self.default_context
from kiara.context import Kiara
from kiara.models.context import ContextSummary
context_config = self.get_context_config(
context_name=context_name, auto_generate=False
)
kiara = Kiara(config=context_config, runtime_config=self.runtime_config)
context_summary = ContextSummary.create_from_context(
kiara=kiara, context_name=context_name
)
if dry_run:
return context_summary
for archive in kiara.get_all_archives().keys():
archive.delete_archive(archive_id=archive.archive_id)
if context_config._context_config_path is not None:
os.unlink(context_config._context_config_path)
return context_summary
find_context_config(self, context_id)
¶Source code in kiara/context/config.py
def find_context_config(self, context_id: uuid.UUID) -> KiaraContextConfig:
raise NotImplementedError()
get_context_config(self, context_name=None, auto_generate=None)
¶Source code in kiara/context/config.py
def get_context_config(
self,
context_name: Union[str, None] = None,
auto_generate: Union[bool, None] = None,
) -> KiaraContextConfig:
if auto_generate is None:
auto_generate = self.auto_generate_contexts
if context_name is None:
context_name = self.default_context
if context_name not in self.available_context_names:
if not auto_generate and not context_name == DEFAULT_CONTEXT_NAME:
raise Exception(
f"No kiara context with name '{context_name}' available."
)
else:
return self.create_context_config(context_alias=context_name)
if context_name in self._context_data.keys():
return self._context_data[context_name]
context_file = self._available_context_files[context_name]
context_data = get_data_from_file(context_file, content_type="yaml")
changed = False
if "extra_pipeline_folders" in context_data.keys():
epf = context_data.pop("extra_pipeline_folders")
context_data.setdefault("extra_pipelines", []).extend(epf)
changed = True
context = KiaraContextConfig(**context_data)
if not changed:
changed = self._validate_context(context_config=context)
if changed:
logger.debug(
"write.context_file",
context_config_file=context_file.as_posix(),
context_name=context_name,
reason="context changed after validation",
)
context_file.parent.mkdir(parents=True, exist_ok=True)
with open(context_file, "wt") as f:
yaml.dump(context.dict(), f)
context._context_config_path = context_file
self._context_data[context_name] = context
return context
load_from_file(path=None)
classmethod
¶Source code in kiara/context/config.py
@classmethod
def load_from_file(cls, path: Union[Path, None] = None) -> "KiaraConfig":
if path is None:
path = Path(KIARA_MAIN_CONFIG_FILE)
if not path.exists():
raise Exception(
f"Can't load kiara config, path does not exist: {path.as_posix()}"
)
if path.is_dir():
path = path / KIARA_CONFIG_FILE_NAME
if not path.exists():
raise Exception(
f"Can't load kiara config, path does not exist: {path.as_posix()}"
)
with path.open("rt") as f:
data = yaml.load(f)
config = KiaraConfig(**data)
config._config_path = path
return config
save(self, path=None)
¶Source code in kiara/context/config.py
def save(self, path: Union[Path, None] = None):
if path is None:
path = Path(KIARA_MAIN_CONFIG_FILE)
if path.exists():
raise Exception(
f"Can't save config file, path already exists: {path.as_posix()}"
)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("wt") as f:
yaml.dump(
self.dict(
exclude={
"context",
"auto_generate_contexts",
"stores_base_path",
"context_search_paths",
"default_context",
"runtime_config",
}
),
f,
)
self._config_path = path
validate_context_search_paths(v)
classmethod
¶Source code in kiara/context/config.py
@validator("context_search_paths")
def validate_context_search_paths(cls, v):
if not v or not v[0]:
v = [KIARA_MAIN_CONTEXTS_PATH]
return v
KiaraContextConfig (BaseModel)
pydantic-model
¶Source code in kiara/context/config.py
class KiaraContextConfig(BaseModel):
class Config:
extra = Extra.forbid
context_id: str = Field(description="A globally unique id for this kiara context.")
archives: Dict[str, KiaraArchiveConfig] = Field(
description="All the archives this kiara context can use and the aliases they are registered with."
)
extra_pipelines: List[str] = Field(
description="Paths to local folders that contain kiara pipelines.",
default_factory=list,
)
_context_config_path: Union[Path, None] = PrivateAttr(default=None)
def add_pipelines(self, *pipelines: str):
for pipeline in pipelines:
if os.path.exists(pipeline):
self.extra_pipelines.append(pipeline)
else:
logger.info(
"ignore.pipeline", reason="path does not exist", path=pipeline
)
# @property
# def db_url(self):
# return get_kiara_db_url(self.context_folder)
#
# @property
# def data_directory(self) -> str:
# return os.path.join(self.context_folder, "data")
archives: Dict[str, kiara.context.config.KiaraArchiveConfig]
pydantic-field
required
¶All the archives this kiara context can use and the aliases they are registered with.
context_id: str
pydantic-field
required
¶A globally unique id for this kiara context.
extra_pipelines: List[str]
pydantic-field
¶Paths to local folders that contain kiara pipelines.
Config
¶Source code in kiara/context/config.py
class Config:
extra = Extra.forbid
add_pipelines(self, *pipelines)
¶Source code in kiara/context/config.py
def add_pipelines(self, *pipelines: str):
for pipeline in pipelines:
if os.path.exists(pipeline):
self.extra_pipelines.append(pipeline)
else:
logger.info(
"ignore.pipeline", reason="path does not exist", path=pipeline
)
KiaraRuntimeConfig (BaseSettings)
pydantic-model
¶Source code in kiara/context/config.py
class KiaraRuntimeConfig(BaseSettings):
class Config:
extra = Extra.forbid
validate_assignment = True
env_prefix = "kiara_runtime_"
job_cache: JobCacheStrategy = Field(
description="Name of the strategy that determines when to re-run jobs or use cached results.",
default=JobCacheStrategy.data_hash,
)
# ignore_errors: bool = Field(
# description="If set, kiara will try to ignore most errors (that can be ignored).",
# default=False,
# )
config_file_settings_source(settings)
¶Source code in kiara/context/config.py
def config_file_settings_source(settings: BaseSettings) -> Dict[str, Any]:
if os.path.isfile(KIARA_MAIN_CONFIG_FILE):
config = get_data_from_file(KIARA_MAIN_CONFIG_FILE, content_type="yaml")
if not isinstance(config, Mapping):
raise ValueError(
f"Invalid config file format, can't parse file: {KIARA_MAIN_CONFIG_FILE}"
)
else:
config = {}
return config
orm
¶
Base: DeclarativeMeta
¶jobs_env_association_table
¶value_env_association_table
¶
AliasOrm (Base)
¶Source code in kiara/context/orm.py
class AliasOrm(Base):
__tablename__ = "aliases"
id: Column[Union[int, None]] = Column(Integer, primary_key=True)
alias: Column[str] = Column(String, index=True, nullable=False)
created: Column[datetime] = Column(UtcDateTime(), nullable=False, index=True)
version: Column[int] = Column(Integer, nullable=False, index=True)
value_id: Column[Union[uuid.UUID, None]] = Column(
UUIDType(binary=True), nullable=True
)
UniqueConstraint(alias, version)
DestinyOrm (Base)
¶Source code in kiara/context/orm.py
class DestinyOrm(Base):
__tablename__ = "destinies"
id: Column[Union[int, None]] = Column(Integer, primary_key=True)
value_id: Column[int] = Column(Integer, ForeignKey("values.id"), nullable=False)
category: Column[str] = Column(String, nullable=False, index=False)
key: Column[str] = Column(String, nullable=False, index=False)
manifest_id: Column[int] = Column(
Integer, ForeignKey("manifests.id"), nullable=False
)
inputs: Column[Union[Dict[Any, Any], List[Any]]] = Column(
JSON, index=False, nullable=False
)
output_name: Column[str] = Column(String, index=False, nullable=False)
destiny_value: Column[Union[int, None]] = Column(
Integer, ForeignKey("values.id"), nullable=True
)
description: Column[Union[str, None]] = Column(String, nullable=True)
UniqueConstraint(value_id, category, key)
EnvironmentOrm (Base)
¶Source code in kiara/context/orm.py
class EnvironmentOrm(Base):
__tablename__ = "environments"
id: Column[Union[int, None]] = Column(Integer, primary_key=True)
metadata_hash: Column[int] = Column(Integer, index=True, nullable=False)
metadata_schema_id = Column(
Integer, ForeignKey("metadata_schema_lookup.id"), nullable=False
)
metadata_payload: Column[Union[Dict[Any, Any], List[Any]]] = Column(
JSON, nullable=False
)
UniqueConstraint(metadata_hash)
JobsOrm (Base)
¶Source code in kiara/context/orm.py
class JobsOrm(Base):
__tablename__ = "jobs"
id: Column[Union[int, None]] = Column(Integer, primary_key=True)
manifest_id: Column[int] = Column(
Integer, ForeignKey("manifests.id"), nullable=False
)
inputs: Column[Union[Dict[Any, Any], List[Any]]] = Column(JSON, nullable=False)
input_hash: Column[str] = Column(String, nullable=False)
is_idempotent: Column[bool] = Column(Boolean, nullable=False)
created: Column[datetime] = Column(UtcDateTime(), default=utcnow(), nullable=False)
started: Column[Union[datetime, None]] = Column(UtcDateTime(), nullable=True)
duration_ms: Column[Union[int, None]] = Column(Integer, nullable=True)
environments = relationship("EnvironmentOrm", secondary=jobs_env_association_table)
ManifestOrm (Base)
¶Source code in kiara/context/orm.py
class ManifestOrm(Base):
__tablename__ = "manifests"
id: Column[Union[int, None]] = Column(Integer, primary_key=True)
module_type: Column[str] = Column(String, index=True, nullable=False)
module_config: Column[Union[Dict[Any, Any], List[Any]]] = Column(
JSON, nullable=False
)
manifest_hash: Column[int] = Column(Integer, index=True, nullable=False)
is_idempotent: Column[bool] = Column(Boolean, nullable=False)
UniqueConstraint(module_type, manifest_hash)
MetadataSchemaOrm (Base)
¶Source code in kiara/context/orm.py
class MetadataSchemaOrm(Base):
__tablename__ = "metadata_schema_lookup"
id: Column[Union[int, None]] = Column(Integer, primary_key=True)
metadata_schema_hash: Column[int] = Column(Integer, index=True, nullable=False)
metadata_type: Column[str] = Column(String, nullable=False)
metadata_schema: Column[Union[Dict[Any, Any], List[Any]]] = Column(
JSON, nullable=False
)
metadata_payloads = relationship("EnvironmentOrm")
UniqueConstraint(metadata_schema_hash)
Pedigree (Base)
¶Source code in kiara/context/orm.py
class Pedigree(Base):
__tablename__ = "pedigrees"
id: Column[Union[int, None]] = Column(Integer, primary_key=True)
manifest_id: Column[int] = Column(
Integer, ForeignKey("manifests.id"), nullable=False
)
inputs: Column[Union[Dict[Any, Any], List[Any]]] = Column(JSON, nullable=False)
ValueOrm (Base)
¶Source code in kiara/context/orm.py
class ValueOrm(Base):
__tablename__ = "values"
id: Column[Union[int, None]] = Column(Integer, primary_key=True)
global_id: Column[uuid.UUID] = Column(UUIDType(binary=True), nullable=False)
data_type_id: Column[int] = Column(
Integer, ForeignKey("data_types.id"), nullable=False
)
data_type_name: Column[str] = Column(String, index=True, nullable=False)
value_size: Column[int] = Column(Integer, index=True, nullable=False)
value_hash: Column[str] = Column(String, index=True, nullable=False)
environments = relationship("EnvironmentOrm", secondary=value_env_association_table)
UniqueConstraint(value_hash, value_size, data_type_id)
ValueTypeOrm (Base)
¶Source code in kiara/context/orm.py
class ValueTypeOrm(Base):
__tablename__ = "data_types"
id: Column[Union[int, None]] = Column(Integer, primary_key=True)
type_config_hash: Column[int] = Column(Integer, index=True, nullable=False)
type_name: Column[str] = Column(String, nullable=False, index=True)
type_config: Column[Union[Dict[Any, Any], List[Any]]] = Column(JSON, nullable=False)
UniqueConstraint(type_config_hash, type_name)
data_types
special
¶
This is the base module that contains everything data type-related in kiara.
I'm still not 100% sure how to best implement the kiara type system, there are several ways it could be done, for example based on Python type-hints, using JSON-schema, Avro (which is my 2nd favourite option), as well as by implementing a custom type-class hierarchy. Which is what I have choosen to try first. For now, it looks like it'll work out, but there is a chance requirements I haven't forseen will crop up that could make this become ugly.
Anyway, the way it works (for now) is that kiara comes with a set of often used data_types (the standard set of: scalars, list, dict, table & array, etc.) which each come with 2 functions that can serialize and deserialize values of that type in a persistant fashion -- which could be storing as a file on disk, or as a cell/row in a database. Those functions will most likley be kiara modules themselves, with even more restricted input/output type options.
In addition, packages that contain modules can implement their own, custom data_types, if suitable ones are not available in core-kiara. Those can either be 'serialized/deserialized' into kiara-native data_types (which in turn will serialize them using their own serializing functions), or will have to implement custom serializing functionality (which will probably be discouraged, since this might not be trivial and there are quite a few things to consider).
TYPE_CONFIG_CLS
¶
TYPE_PYTHON_CLS
¶
logger
¶
Classes¶
DataType (ABC, Generic)
¶
Base class that all kiara data_types must inherit from.
kiara data_types have 3 main responsibilities:
- serialize into / deserialize from persistent state
- data validation
- metadata extraction
Serializing being the arguably most important of those, because without most of the data management features of kiara would be impossible. Validation should not require any explanation. Metadata extraction is important, because that metadata will be available to other components of kiara (or frontends for it), without them having to request the actual data. That will hopefully make kiara very efficient in terms of memory management, as well as data transfer and I/O. Ideally, the actual data (bytes) will only be requested at the last possible moment. For example when a module needs the input data to do processing on it -- and even then it might be that it only requests a part of the data, say a single column of a table. Or when a frontend needs to display/visualize the data.
Source code in kiara/data_types/__init__.py
class DataType(abc.ABC, Generic[TYPE_PYTHON_CLS, TYPE_CONFIG_CLS]):
"""Base class that all *kiara* data_types must inherit from.
*kiara* data_types have 3 main responsibilities:
- serialize into / deserialize from persistent state
- data validation
- metadata extraction
Serializing being the arguably most important of those, because without most of the data management features of
*kiara* would be impossible. Validation should not require any explanation. Metadata extraction is important, because
that metadata will be available to other components of *kiara* (or frontends for it), without them having to request
the actual data. That will hopefully make *kiara* very efficient in terms of memory management, as well as data
transfer and I/O. Ideally, the actual data (bytes) will only be requested at the last possible moment. For example when a
module needs the input data to do processing on it -- and even then it might be that it only requests a part of the
data, say a single column of a table. Or when a frontend needs to display/visualize the data.
"""
@classmethod
def retrieve_available_type_profiles(cls) -> Mapping[str, Mapping[str, Any]]:
return {}
@classmethod
@abc.abstractmethod
def python_class(cls) -> Type[TYPE_PYTHON_CLS]:
pass
@classmethod
def data_type_config_class(cls) -> Type[TYPE_CONFIG_CLS]:
return DataTypeConfig # type: ignore
@classmethod
def _calculate_data_type_hash(
cls, data_type_config: Union[Mapping[str, Any], DataTypeConfig]
) -> int:
if isinstance(data_type_config, Mapping):
data_type_config = cls.data_type_config_class()(**data_type_config) # type: ignore
obj = {
"type": cls._data_type_name, # type: ignore
"type_config": data_type_config.config_hash,
}
h = DeepHash(obj, hasher=KIARA_HASH_FUNCTION)
return h[obj]
def __init__(self, **type_config: Any):
try:
self._type_config: TYPE_CONFIG_CLS = self.__class__.data_type_config_class()(**type_config) # type: ignore # TODO: double-check this is only a mypy issue
except ValidationError as ve:
raise ValueTypeConfigException(
f"Error creating object for type: {ve}",
self.__class__,
type_config,
ve,
)
self._data_type_hash: Union[int, None] = None
self._characteristics: Union[DataTypeCharacteristics, None] = None
self._info: Union[DataTypeInfo, None] = None
@property
def data_type_name(self) -> str:
return self._data_type_name # type: ignore
@property
def data_type_hash(self) -> int:
if self._data_type_hash is None:
self._data_type_hash = self.__class__._calculate_data_type_hash(
self._type_config
)
return self._data_type_hash
@property
def info(self) -> "DataTypeInfo":
if self._info is not None:
return self._info
from kiara.models.values.value import DataTypeInfo
self._info = DataTypeInfo(
data_type_name=self.data_type_name,
data_type_config=self.type_config.dict(),
characteristics=self.characteristics,
data_type_class=PythonClass.from_class(self.__class__),
)
self._info._data_type_instance = self
return self._info
@property
def characteristics(self) -> DataTypeCharacteristics:
if self._characteristics is not None:
return self._characteristics
self._characteristics = self._retrieve_characteristics()
return self._characteristics
def _retrieve_characteristics(self) -> DataTypeCharacteristics:
return DataTypeCharacteristics()
# @abc.abstractmethod
# def is_immutable(self) -> bool:
# pass
def calculate_hash(self, data: "SerializedData") -> str:
"""Calculate the hash of the value."""
return data.instance_id
def calculate_size(self, data: "SerializedData") -> int:
"""Calculate the size of the value."""
return data.data_size
def serialize_as_json(self, data: Any) -> "SerializedData":
_data = {"data": {"type": "inline-json", "inline_data": data, "codec": "json"}}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "json",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_type": "deserialize.from_json",
"module_config": {"result_path": "data"},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
def serialize(self, data: TYPE_PYTHON_CLS) -> Union[None, str, "SerializedData"]:
logger.debug(
"ignore.serialize_request",
data_type=self.data_type_name,
reason="no 'serialize' method imnplemented",
)
return NO_SERIALIZATION_MARKER
# raise NotImplementedError(f"Data type '{self.data_type_name}' does not support serialization.")
#
# try:
# import pickle5 as pickle
# except Exception:
# import pickle # type: ignore
#
# pickled = pickle.dumps(data, protocol=5)
# _data = {"python_object": {"type": "chunk", "chunk": pickled, "codec": "raw"}}
#
# serialized_data = {
# "data_type": self.data_type_name,
# "data_type_config": self.type_config.dict(),
# "data": _data,
# "serialization_profile": "pickle",
# "serialization_metadata": {
# "profile": "pickle",
# "environment": {},
# "deserialize": {
# "object": {
# "module_name": "value.unpickle",
# "module_config": {
# "value_type": "any",
# "target_profile": "object",
# "serialization_profile": "pickle",
# },
# }
# },
# },
# }
# from kiara.models.values.value import SerializationResult
#
# serialized = SerializationResult(**serialized_data)
# return serialized
@property
def type_config(self) -> TYPE_CONFIG_CLS:
return self._type_config
def _pre_examine_data(
self, data: Any, schema: ValueSchema
) -> Tuple[Any, Union[str, "SerializedData"], ValueStatus, str, int]:
assert data is not None
if data is SpecialValue.NOT_SET:
status = ValueStatus.NOT_SET
data = None
elif data is SpecialValue.NO_VALUE:
status = ValueStatus.NONE
data = None
else:
status = ValueStatus.SET
# if data is None and schema.default not in [
# None,
# SpecialValue.NO_VALUE,
# SpecialValue.NOT_SET,
# ]:
#
# status = ValueStatus.DEFAULT
# if callable(schema.default):
# data = schema.default()
# else:
# data = copy.deepcopy(schema.default)
if data is None or data is SpecialValue.NOT_SET:
# if schema.default in [None, SpecialValue.NO_VALUE]:
# data = SpecialValue.NO_VALUE
# status = ValueStatus.NONE
# elif schema.default == SpecialValue.NOT_SET:
# data = SpecialValue.NOT_SET
# status = ValueStatus.NOT_SET
size = 0
value_hash = INVALID_HASH_MARKER
serialized: Union[None, str, "SerializedData"] = NO_SERIALIZATION_MARKER
else:
from kiara.models.values.value import SerializedData
if isinstance(data, SerializedData):
# TODO: assert value is in schema lineage
# assert data.data_type == schema.type
# assert data.data_type_config == schema.type_config
serialized = data
not_serialized: bool = False
else:
data = self.parse_python_obj(data)
if data is None:
raise Exception(
f"Invalid data, can't parse into a value of type '{schema.type}'."
)
self._validate(data)
serialized = self.serialize(data)
if serialized is None:
serialized = NO_SERIALIZATION_MARKER
if isinstance(serialized, str):
not_serialized = True
else:
not_serialized = False
if not_serialized:
size = INVALID_SIZE_MARKER
value_hash = INVALID_HASH_MARKER
else:
size = serialized.data_size # type: ignore
value_hash = serialized.instance_id # type: ignore
assert serialized is not None
result = (data, serialized, status, value_hash, size)
return result
def assemble_value(
self,
value_id: uuid.UUID,
data: Any,
schema: ValueSchema,
environment_hashes: Mapping[str, Mapping[str, str]],
serialized: Union[str, "SerializedData"],
status: Union[ValueStatus, str],
value_hash: str,
value_size: int,
pedigree: "ValuePedigree",
kiara_id: uuid.UUID,
pedigree_output_name: str,
) -> Tuple["Value", Any]:
from kiara.models.values.value import Value
if isinstance(status, str):
status = ValueStatus(status).name
if status in [ValueStatus.SET, ValueStatus.DEFAULT]:
try:
value = Value(
value_id=value_id,
kiara_id=kiara_id,
value_status=status,
value_size=value_size,
value_hash=value_hash,
value_schema=schema,
environment_hashes=environment_hashes,
pedigree=pedigree,
pedigree_output_name=pedigree_output_name,
data_type_info=self.info,
)
except Exception as e:
raise KiaraValueException(
data_type=self.__class__, value_data=data, exception=e
)
else:
value = Value(
value_id=value_id,
kiara_id=kiara_id,
value_status=status,
value_size=value_size,
value_hash=value_hash,
value_schema=schema,
environment_hashes=environment_hashes,
pedigree=pedigree,
pedigree_output_name=pedigree_output_name,
data_type_info=self.info,
)
value._value_data = data
value._serialized_data = serialized
return value, data
def parse_python_obj(self, data: Any) -> TYPE_PYTHON_CLS:
"""Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object).
If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to
avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Arguments:
v: the value
Returns:
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object
"""
return data
def _validate(self, value: TYPE_PYTHON_CLS) -> None:
"""Validate the value. This expects an instance of the defined Python class (from 'backing_python_type)."""
if not isinstance(value, self.__class__.python_class()):
raise ValueError(
f"Invalid python type '{type(value)}', must be: {self.__class__.python_class()}"
)
def create_renderable(self, **config):
show_type_info = config.get("show_type_info", False)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key")
table.add_column("value", style="i")
table.add_row("type_name", self.data_type_name)
config_json = self.type_config.json(
exclude_unset=True, option=orjson.OPT_INDENT_2
)
config = Syntax(config_json, "json", background_color="default")
table.add_row("type_config", config)
if show_type_info:
from kiara.interfaces.python_api.models.info import DataTypeClassInfo
info = DataTypeClassInfo.create_from_type_class(self.__class__)
table.add_row("", "")
table.add_row("", Rule())
table.add_row("type_info", info)
return table
characteristics: DataTypeCharacteristics
property
readonly
¶data_type_hash: int
property
readonly
¶data_type_name: str
property
readonly
¶info: DataTypeInfo
property
readonly
¶type_config: ~TYPE_CONFIG_CLS
property
readonly
¶Methods¶
assemble_value(self, value_id, data, schema, environment_hashes, serialized, status, value_hash, value_size, pedigree, kiara_id, pedigree_output_name)
¶Source code in kiara/data_types/__init__.py
def assemble_value(
self,
value_id: uuid.UUID,
data: Any,
schema: ValueSchema,
environment_hashes: Mapping[str, Mapping[str, str]],
serialized: Union[str, "SerializedData"],
status: Union[ValueStatus, str],
value_hash: str,
value_size: int,
pedigree: "ValuePedigree",
kiara_id: uuid.UUID,
pedigree_output_name: str,
) -> Tuple["Value", Any]:
from kiara.models.values.value import Value
if isinstance(status, str):
status = ValueStatus(status).name
if status in [ValueStatus.SET, ValueStatus.DEFAULT]:
try:
value = Value(
value_id=value_id,
kiara_id=kiara_id,
value_status=status,
value_size=value_size,
value_hash=value_hash,
value_schema=schema,
environment_hashes=environment_hashes,
pedigree=pedigree,
pedigree_output_name=pedigree_output_name,
data_type_info=self.info,
)
except Exception as e:
raise KiaraValueException(
data_type=self.__class__, value_data=data, exception=e
)
else:
value = Value(
value_id=value_id,
kiara_id=kiara_id,
value_status=status,
value_size=value_size,
value_hash=value_hash,
value_schema=schema,
environment_hashes=environment_hashes,
pedigree=pedigree,
pedigree_output_name=pedigree_output_name,
data_type_info=self.info,
)
value._value_data = data
value._serialized_data = serialized
return value, data
calculate_hash(self, data)
¶Calculate the hash of the value.
Source code in kiara/data_types/__init__.py
def calculate_hash(self, data: "SerializedData") -> str:
"""Calculate the hash of the value."""
return data.instance_id
calculate_size(self, data)
¶Calculate the size of the value.
Source code in kiara/data_types/__init__.py
def calculate_size(self, data: "SerializedData") -> int:
"""Calculate the size of the value."""
return data.data_size
create_renderable(self, **config)
¶Source code in kiara/data_types/__init__.py
def create_renderable(self, **config):
show_type_info = config.get("show_type_info", False)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key")
table.add_column("value", style="i")
table.add_row("type_name", self.data_type_name)
config_json = self.type_config.json(
exclude_unset=True, option=orjson.OPT_INDENT_2
)
config = Syntax(config_json, "json", background_color="default")
table.add_row("type_config", config)
if show_type_info:
from kiara.interfaces.python_api.models.info import DataTypeClassInfo
info = DataTypeClassInfo.create_from_type_class(self.__class__)
table.add_row("", "")
table.add_row("", Rule())
table.add_row("type_info", info)
return table
data_type_config_class()
classmethod
¶Source code in kiara/data_types/__init__.py
@classmethod
def data_type_config_class(cls) -> Type[TYPE_CONFIG_CLS]:
return DataTypeConfig # type: ignore
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
~TYPE_PYTHON_CLS |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/__init__.py
def parse_python_obj(self, data: Any) -> TYPE_PYTHON_CLS:
"""Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object).
If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to
avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Arguments:
v: the value
Returns:
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object
"""
return data
python_class()
classmethod
¶Source code in kiara/data_types/__init__.py
@classmethod
@abc.abstractmethod
def python_class(cls) -> Type[TYPE_PYTHON_CLS]:
pass
retrieve_available_type_profiles()
classmethod
¶Source code in kiara/data_types/__init__.py
@classmethod
def retrieve_available_type_profiles(cls) -> Mapping[str, Mapping[str, Any]]:
return {}
serialize(self, data)
¶Source code in kiara/data_types/__init__.py
def serialize(self, data: TYPE_PYTHON_CLS) -> Union[None, str, "SerializedData"]:
logger.debug(
"ignore.serialize_request",
data_type=self.data_type_name,
reason="no 'serialize' method imnplemented",
)
return NO_SERIALIZATION_MARKER
# raise NotImplementedError(f"Data type '{self.data_type_name}' does not support serialization.")
#
# try:
# import pickle5 as pickle
# except Exception:
# import pickle # type: ignore
#
# pickled = pickle.dumps(data, protocol=5)
# _data = {"python_object": {"type": "chunk", "chunk": pickled, "codec": "raw"}}
#
# serialized_data = {
# "data_type": self.data_type_name,
# "data_type_config": self.type_config.dict(),
# "data": _data,
# "serialization_profile": "pickle",
# "serialization_metadata": {
# "profile": "pickle",
# "environment": {},
# "deserialize": {
# "object": {
# "module_name": "value.unpickle",
# "module_config": {
# "value_type": "any",
# "target_profile": "object",
# "serialization_profile": "pickle",
# },
# }
# },
# },
# }
# from kiara.models.values.value import SerializationResult
#
# serialized = SerializationResult(**serialized_data)
# return serialized
serialize_as_json(self, data)
¶Source code in kiara/data_types/__init__.py
def serialize_as_json(self, data: Any) -> "SerializedData":
_data = {"data": {"type": "inline-json", "inline_data": data, "codec": "json"}}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "json",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_type": "deserialize.from_json",
"module_config": {"result_path": "data"},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
DataTypeConfig (BaseModel)
pydantic-model
¶
Base class that describes the configuration a [DataType][kiara.data.data_types.DataType] class accepts.
This is stored in the _config_cls class attribute in each DataType class. By default,
a DataType is not configurable, unless the _config_cls class attribute points to a sub-class of this class.
Source code in kiara/data_types/__init__.py
class DataTypeConfig(BaseModel):
"""Base class that describes the configuration a [``DataType``][kiara.data.data_types.DataType] class accepts.
This is stored in the ``_config_cls`` class attribute in each ``DataType`` class. By default,
a ``DataType`` is not configurable, unless the ``_config_cls`` class attribute points to a sub-class of this class.
"""
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
extra = Extra.forbid
@classmethod
def requires_config(cls) -> bool:
"""Return whether this class can be used as-is, or requires configuration before an instance can be created."""
for field_name, field in cls.__fields__.items():
if field.required and field.default is None:
return True
return False
_config_hash: Union[int, None] = PrivateAttr(default=None)
def get(self, key: str) -> Any:
"""Get the value for the specified configuation key."""
if key not in self.__fields__:
raise Exception(
f"No config value '{key}' in module config class '{self.__class__.__name__}'."
)
return getattr(self, key)
@property
def config_hash(self) -> int:
if self._config_hash is None:
_d = self.dict()
hashes = DeepHash(_d)
self._config_hash = hashes[_d]
return self._config_hash
def __eq__(self, other):
if self.__class__ != other.__class__:
return False
return self.dict() == other.dict()
def __hash__(self):
return self.config_hash
def __rich_console__(
self, console: Console, options: ConsoleOptions
) -> RenderResult:
my_table = Table(box=box.MINIMAL, show_header=False)
my_table.add_column("Field name", style="i")
my_table.add_column("Value")
for field in self.__fields__:
my_table.add_row(field, getattr(self, field))
yield my_table
config_hash: int
property
readonly
¶
Config
¶Source code in kiara/data_types/__init__.py
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
extra = Extra.forbid
extra
¶json_loads
¶json_dumps(v, *, default=None, **args)
¶Source code in kiara/data_types/__init__.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
Methods¶
get(self, key)
¶Get the value for the specified configuation key.
Source code in kiara/data_types/__init__.py
def get(self, key: str) -> Any:
"""Get the value for the specified configuation key."""
if key not in self.__fields__:
raise Exception(
f"No config value '{key}' in module config class '{self.__class__.__name__}'."
)
return getattr(self, key)
requires_config()
classmethod
¶Return whether this class can be used as-is, or requires configuration before an instance can be created.
Source code in kiara/data_types/__init__.py
@classmethod
def requires_config(cls) -> bool:
"""Return whether this class can be used as-is, or requires configuration before an instance can be created."""
for field_name, field in cls.__fields__.items():
if field.required and field.default is None:
return True
return False
Modules¶
included_core_types
special
¶
KIARA_MODEL_CLS
¶SCALAR_CHARACTERISTICS
¶Classes¶
AnyType (DataType, Generic)
¶'Any' type, the parent type for most other types.
This type acts as the parents for all (or at least most) non-internal value types. There are some generic operations (like 'persist_value', or 'pretty_print') which are implemented for this type, so it's descendents have a fallback option in case no subtype-specific operations are implemented for it. In general, it is not recommended to use the 'any' type as module input or output, but it is possible. Values of type 'any' are not allowed to be persisted (at the moment, this might or might not change).
Source code in kiara/data_types/included_core_types/__init__.py
class AnyType(
DataType[TYPE_PYTHON_CLS, DataTypeConfig], Generic[TYPE_PYTHON_CLS, TYPE_CONFIG_CLS]
):
"""'Any' type, the parent type for most other types.
This type acts as the parents for all (or at least most) non-internal value types. There are some generic operations
(like 'persist_value', or 'pretty_print') which are implemented for this type, so it's descendents have a fallback
option in case no subtype-specific operations are implemented for it. In general, it is not recommended to use the 'any'
type as module input or output, but it is possible. Values of type 'any' are not allowed to be persisted (at the moment,
this might or might not change).
"""
_data_type_name = "any"
@classmethod
def python_class(cls) -> Type:
return object
def pretty_print_as__string(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
if hasattr(self, "_pretty_print_as__string"):
return self._pretty_print_as__string(value=value, render_config=render_config) # type: ignore
return str(value.data)
def pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
):
if hasattr(self, "_pretty_print_as__terminal_renderable"):
return self._pretty_print_as__terminal_renderable(value=value, render_config=render_config) # type: ignore
data = value.data
from pydantic import BaseModel
if isinstance(data, BaseModel):
from kiara.utils.output import create_table_from_model_object
rendered = create_table_from_model_object(
model=data, render_config=render_config
)
elif isinstance(data, Iterable):
import pprint
rendered = pprint.pformat(data)
else:
rendered = str(data)
return rendered
def render_as__string(
self, value: "Value", render_config: Mapping[str, Any], manifest: "Manifest"
):
if hasattr(self, "_render_as__string"):
return self._render_as__string(value=value, render_scene=render_config, manifest=manifest) # type: ignore
else:
return self.pretty_print_as__string(value=value, render_config={})
def render_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any], manifest: "Manifest"
):
if hasattr(self, "_render_as__terminal_renderable"):
return self._render_as__terminal(value=value, render_config=render_config, manifest=manifest) # type: ignore
return self.render_as__string(
value=value, render_config=render_config, manifest=manifest
)
pretty_print_as__string(self, value, render_config)
¶Source code in kiara/data_types/included_core_types/__init__.py
def pretty_print_as__string(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
if hasattr(self, "_pretty_print_as__string"):
return self._pretty_print_as__string(value=value, render_config=render_config) # type: ignore
return str(value.data)
pretty_print_as__terminal_renderable(self, value, render_config)
¶Source code in kiara/data_types/included_core_types/__init__.py
def pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
):
if hasattr(self, "_pretty_print_as__terminal_renderable"):
return self._pretty_print_as__terminal_renderable(value=value, render_config=render_config) # type: ignore
data = value.data
from pydantic import BaseModel
if isinstance(data, BaseModel):
from kiara.utils.output import create_table_from_model_object
rendered = create_table_from_model_object(
model=data, render_config=render_config
)
elif isinstance(data, Iterable):
import pprint
rendered = pprint.pformat(data)
else:
rendered = str(data)
return rendered
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/__init__.py
@classmethod
def python_class(cls) -> Type:
return object
render_as__string(self, value, render_config, manifest)
¶Source code in kiara/data_types/included_core_types/__init__.py
def render_as__string(
self, value: "Value", render_config: Mapping[str, Any], manifest: "Manifest"
):
if hasattr(self, "_render_as__string"):
return self._render_as__string(value=value, render_scene=render_config, manifest=manifest) # type: ignore
else:
return self.pretty_print_as__string(value=value, render_config={})
render_as__terminal_renderable(self, value, render_config, manifest)
¶Source code in kiara/data_types/included_core_types/__init__.py
def render_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any], manifest: "Manifest"
):
if hasattr(self, "_render_as__terminal_renderable"):
return self._render_as__terminal(value=value, render_config=render_config, manifest=manifest) # type: ignore
return self.render_as__string(
value=value, render_config=render_config, manifest=manifest
)
BooleanType (AnyType)
¶A boolean.
Source code in kiara/data_types/included_core_types/__init__.py
class BooleanType(AnyType[bool, DataTypeConfig]):
"A boolean."
_data_type_name = "boolean"
@classmethod
def python_class(cls) -> Type:
return bool
def serialize(self, data: bool) -> "SerializedData":
result = self.serialize_as_json(data)
return result
def _retrieve_characteristics(self) -> DataTypeCharacteristics:
return SCALAR_CHARACTERISTICS
# def calculate_size(self, data: bool) -> int:
# return 24
#
# def calculate_hash(cls, data: bool) -> int:
# return 1 if data else 0
def parse_python_obj(self, data: Any) -> bool:
if data is True or data is False:
return data
elif data == 0:
return False
elif data == 1:
return True
elif isinstance(data, str):
if data.lower() == "true":
return True
elif data.lower() == "false":
return False
raise Exception(f"Can't parse value '{data}' as boolean.")
def validate(cls, value: Any):
pass
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
bool |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/__init__.py
def parse_python_obj(self, data: Any) -> bool:
if data is True or data is False:
return data
elif data == 0:
return False
elif data == 1:
return True
elif isinstance(data, str):
if data.lower() == "true":
return True
elif data.lower() == "false":
return False
raise Exception(f"Can't parse value '{data}' as boolean.")
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/__init__.py
@classmethod
def python_class(cls) -> Type:
return bool
serialize(self, data)
¶Source code in kiara/data_types/included_core_types/__init__.py
def serialize(self, data: bool) -> "SerializedData":
result = self.serialize_as_json(data)
return result
validate(cls, value)
¶Source code in kiara/data_types/included_core_types/__init__.py
def validate(cls, value: Any):
pass
BytesType (AnyType)
¶An array of bytes.
Source code in kiara/data_types/included_core_types/__init__.py
class BytesType(AnyType[bytes, DataTypeConfig]):
"""An array of bytes."""
_data_type_name = "bytes"
@classmethod
def python_class(cls) -> Type:
return bytes
def serialize(self, data: bytes) -> "SerializedData":
_data = {"bytes": {"type": "chunk", "chunk": data, "codec": "raw"}}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "raw",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_name": "load.bytes",
"module_config": {
"value_type": "bytes",
"target_profile": "python_object",
"serialization_profile": "raw",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
def _pretty_print_as__string(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
data: bytes = value.data
return data.decode()
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/__init__.py
@classmethod
def python_class(cls) -> Type:
return bytes
serialize(self, data)
¶Source code in kiara/data_types/included_core_types/__init__.py
def serialize(self, data: bytes) -> "SerializedData":
_data = {"bytes": {"type": "chunk", "chunk": data, "codec": "raw"}}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "raw",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_name": "load.bytes",
"module_config": {
"value_type": "bytes",
"target_profile": "python_object",
"serialization_profile": "raw",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
DictValueType (AnyType)
¶A dictionary.
In addition to the actual dictionary value, this value type comes also with an optional schema, describing the dictionary. In case no schema was attached, a simple generic one is attached. This data type is backed by the [DictModel][kiara_plugin.core_types.models.DictModel] class.
Source code in kiara/data_types/included_core_types/__init__.py
class DictValueType(AnyType[DictModel, DataTypeConfig]):
"""A dictionary.
In addition to the actual dictionary value, this value type comes also with an optional schema, describing the
dictionary. In case no schema was attached, a simple generic one is attached. This data type is backed by the
[DictModel][kiara_plugin.core_types.models.DictModel] class.
"""
_data_type_name = "dict"
@classmethod
def python_class(cls) -> Type:
return DictModel
# def calculate_size(self, data: DictModel) -> int:
# return data.size
#
# def calculate_hash(self, data: DictModel) -> int:
# return data.value_hash
def _retrieve_characteristics(self) -> DataTypeCharacteristics:
return DataTypeCharacteristics(is_scalar=False, is_json_serializable=True)
def parse_python_obj(self, data: Any) -> DictModel:
python_cls = data.__class__
dict_data = None
schema = None
if isinstance(data, Mapping):
if (
len(data) == 3
and "dict_data" in data.keys()
and "data_schema" in data.keys()
and "python_class" in data.keys()
):
dict_model = DictModel(
dict_data=data["dict_data"],
data_schema=data["data_schema"],
python_class=data["python_class"],
)
return dict_model
schema = {"title": "dict", "type": "object"}
dict_data = data
elif isinstance(data, BaseModel):
dict_data = data.dict()
schema = data.schema()
elif isinstance(data, str):
try:
dict_data = orjson.loads(data)
schema = {"title": "dict", "type": "object"}
except Exception:
pass
if dict_data is None or schema is None:
raise Exception(f"Invalid data for value type 'dict': {data}")
result = {
"dict_data": dict_data,
"data_schema": schema,
"python_class": PythonClass.from_class(python_cls).dict(),
}
return DictModel(**result)
def _validate(self, data: DictModel) -> None:
if not isinstance(data, DictModel):
raise Exception(f"Invalid type: {type(data)}.")
# def render_as__string(self, value: Value, render_config: Mapping[str, Any]) -> str:
#
# data: DictModel = value.data
# return orjson_dumps(data.dict_data, option=orjson.OPT_INDENT_2)
def _pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
):
show_schema = render_config.get("show_schema", True)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
data: DictModel = value.data
data_json = orjson_dumps(data.dict_data, option=orjson.OPT_INDENT_2)
table.add_row(
"dict data", Syntax(data_json, "json", background_color="default")
)
if show_schema:
schema_json = orjson_dumps(data.data_schema, option=orjson.OPT_INDENT_2)
table.add_row(
"dict schema", Syntax(schema_json, "json", background_color="default")
)
return table
def serialize(self, data: DictModel) -> "SerializedData":
result = self.serialize_as_json(data.dict())
return result
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
DictModel |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/__init__.py
def parse_python_obj(self, data: Any) -> DictModel:
python_cls = data.__class__
dict_data = None
schema = None
if isinstance(data, Mapping):
if (
len(data) == 3
and "dict_data" in data.keys()
and "data_schema" in data.keys()
and "python_class" in data.keys()
):
dict_model = DictModel(
dict_data=data["dict_data"],
data_schema=data["data_schema"],
python_class=data["python_class"],
)
return dict_model
schema = {"title": "dict", "type": "object"}
dict_data = data
elif isinstance(data, BaseModel):
dict_data = data.dict()
schema = data.schema()
elif isinstance(data, str):
try:
dict_data = orjson.loads(data)
schema = {"title": "dict", "type": "object"}
except Exception:
pass
if dict_data is None or schema is None:
raise Exception(f"Invalid data for value type 'dict': {data}")
result = {
"dict_data": dict_data,
"data_schema": schema,
"python_class": PythonClass.from_class(python_cls).dict(),
}
return DictModel(**result)
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/__init__.py
@classmethod
def python_class(cls) -> Type:
return DictModel
serialize(self, data)
¶Source code in kiara/data_types/included_core_types/__init__.py
def serialize(self, data: DictModel) -> "SerializedData":
result = self.serialize_as_json(data.dict())
return result
KiaraModelValueType (AnyType, Generic)
¶A value type that is used internally.
This type should not be used by user-facing modules and/or operations.
Source code in kiara/data_types/included_core_types/__init__.py
class KiaraModelValueType(
AnyType[KIARA_MODEL_CLS, TYPE_CONFIG_CLS], Generic[KIARA_MODEL_CLS, TYPE_CONFIG_CLS]
):
"""A value type that is used internally.
This type should not be used by user-facing modules and/or operations.
"""
_data_type_name = None # type: ignore
@classmethod
def data_type_config_class(cls) -> Type[DataTypeConfig]:
return DataTypeConfig
@abc.abstractmethod
def create_model_from_python_obj(self, data: Any) -> KIARA_MODEL_CLS:
pass
def parse_python_obj(self, data: Any) -> KIARA_MODEL_CLS:
if isinstance(data, self.__class__.python_class()):
return data # type: ignore
data = self.create_model_from_python_obj(data)
return data
def _validate(self, data: KiaraModel) -> None:
if not isinstance(data, self.__class__.python_class()):
raise Exception(
f"Invalid type '{type(data)}', must be: {self.__class__.python_class().__name__}, or subclass."
)
create_model_from_python_obj(self, data)
¶Source code in kiara/data_types/included_core_types/__init__.py
@abc.abstractmethod
def create_model_from_python_obj(self, data: Any) -> KIARA_MODEL_CLS:
pass
data_type_config_class()
classmethod
¶Source code in kiara/data_types/included_core_types/__init__.py
@classmethod
def data_type_config_class(cls) -> Type[DataTypeConfig]:
return DataTypeConfig
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
~KIARA_MODEL_CLS |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/__init__.py
def parse_python_obj(self, data: Any) -> KIARA_MODEL_CLS:
if isinstance(data, self.__class__.python_class()):
return data # type: ignore
data = self.create_model_from_python_obj(data)
return data
NoneType (DataType)
¶Type indicating a 'None' value
Source code in kiara/data_types/included_core_types/__init__.py
class NoneType(DataType[SpecialValue, DataTypeConfig]):
"""Type indicating a 'None' value"""
_data_type_name = "none"
@classmethod
def python_class(cls) -> Type:
return SpecialValue
# def is_immutable(self) -> bool:
# return False
def calculate_hash(self, data: Any) -> str:
return INVALID_HASH_MARKER
def calculate_size(self, data: Any) -> int:
return 0
def parse_python_obj(self, data: Any) -> SpecialValue:
return SpecialValue.NO_VALUE
def pretty_print_as__string(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
return "None"
def pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
):
return "None"
calculate_hash(self, data)
¶Calculate the hash of the value.
Source code in kiara/data_types/included_core_types/__init__.py
def calculate_hash(self, data: Any) -> str:
return INVALID_HASH_MARKER
calculate_size(self, data)
¶Calculate the size of the value.
Source code in kiara/data_types/included_core_types/__init__.py
def calculate_size(self, data: Any) -> int:
return 0
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
SpecialValue |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/__init__.py
def parse_python_obj(self, data: Any) -> SpecialValue:
return SpecialValue.NO_VALUE
pretty_print_as__string(self, value, render_config)
¶Source code in kiara/data_types/included_core_types/__init__.py
def pretty_print_as__string(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
return "None"
pretty_print_as__terminal_renderable(self, value, render_config)
¶Source code in kiara/data_types/included_core_types/__init__.py
def pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
):
return "None"
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/__init__.py
@classmethod
def python_class(cls) -> Type:
return SpecialValue
StringType (AnyType)
¶A string.
Source code in kiara/data_types/included_core_types/__init__.py
class StringType(AnyType[str, DataTypeConfig]):
"""A string."""
_data_type_name = "string"
@classmethod
def python_class(cls) -> Type:
return str
def serialize(self, data: str) -> "SerializedData":
_data = {
"string": {"type": "chunk", "chunk": data.encode("utf-8"), "codec": "raw"}
}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "raw",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_type": "load.string",
"module_config": {
"value_type": "string",
"target_profile": "python_object",
"serialization_profile": "raw",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
def _retrieve_characteristics(self) -> DataTypeCharacteristics:
return SCALAR_CHARACTERISTICS
def _validate(cls, value: Any) -> None:
if not isinstance(value, str):
raise ValueError(f"Invalid type '{type(value)}': string required")
def pretty_print_as__bytes(self, value: "Value", render_config: Mapping[str, Any]):
value_str: str = value.data
return value_str.encode()
pretty_print_as__bytes(self, value, render_config)
¶Source code in kiara/data_types/included_core_types/__init__.py
def pretty_print_as__bytes(self, value: "Value", render_config: Mapping[str, Any]):
value_str: str = value.data
return value_str.encode()
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/__init__.py
@classmethod
def python_class(cls) -> Type:
return str
serialize(self, data)
¶Source code in kiara/data_types/included_core_types/__init__.py
def serialize(self, data: str) -> "SerializedData":
_data = {
"string": {"type": "chunk", "chunk": data.encode("utf-8"), "codec": "raw"}
}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "raw",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_type": "load.string",
"module_config": {
"value_type": "string",
"target_profile": "python_object",
"serialization_profile": "raw",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
Modules¶
filesystem
¶SUPPORTED_FILE_TYPES
¶logger
¶
FileBundleValueType (AnyType)
¶A bundle of files (like a folder, zip archive, etc.).
Source code in kiara/data_types/included_core_types/filesystem.py
class FileBundleValueType(AnyType[FileBundle, FileTypeConfig]):
"""A bundle of files (like a folder, zip archive, etc.)."""
_data_type_name = "file_bundle"
@classmethod
def retrieve_available_type_profiles(cls) -> Mapping[str, Mapping[str, Any]]:
result = {}
for ft in SUPPORTED_FILE_TYPES:
result[f"{ft}_file_bundle"] = {"content_type": ft}
return result
@classmethod
def python_class(cls) -> Type:
return FileBundle
@classmethod
def data_type_config_class(cls) -> Type[FileTypeConfig]:
return FileTypeConfig
def serialize(self, data: FileBundle) -> "SerializedData":
file_data: Dict[str, Any] = {}
file_metadata = {}
for rel_path, file in data.included_files.items():
file_data[rel_path] = {"type": "file", "codec": "raw", "file": file.path}
file_metadata[rel_path] = {
"file_name": file.file_name,
# "import_time": file.import_time,
}
metadata: Dict[str, Any] = {
"included_files": file_metadata,
"bundle_name": data.bundle_name,
# "import_time": data.import_time,
"size": data.size,
"number_of_files": data.number_of_files,
}
assert "__file_metadata__" not in file_data
file_data["__file_metadata__"] = {
"type": "inline-json",
"codec": "json",
"inline_data": metadata,
}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": file_data,
"serialization_profile": "copy",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_type": "deserialize.file_bundle",
"module_config": {
"value_type": "file_bundle",
"target_profile": "python_object",
"serialization_profile": "copy",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
def parse_python_obj(self, data: Any) -> FileBundle:
if isinstance(data, FileBundle):
return data
elif isinstance(data, str):
return FileBundle.import_folder(source=data)
else:
raise Exception(
f"Can't create FileBundle from data of type '{type(data)}'."
)
def _pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
bundle: FileBundle = value.data
renderable = bundle.create_renderable(**render_config)
return renderable
data_type_config_class()
classmethod
¶Source code in kiara/data_types/included_core_types/filesystem.py
@classmethod
def data_type_config_class(cls) -> Type[FileTypeConfig]:
return FileTypeConfig
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
FileBundle |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/filesystem.py
def parse_python_obj(self, data: Any) -> FileBundle:
if isinstance(data, FileBundle):
return data
elif isinstance(data, str):
return FileBundle.import_folder(source=data)
else:
raise Exception(
f"Can't create FileBundle from data of type '{type(data)}'."
)
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/filesystem.py
@classmethod
def python_class(cls) -> Type:
return FileBundle
retrieve_available_type_profiles()
classmethod
¶Source code in kiara/data_types/included_core_types/filesystem.py
@classmethod
def retrieve_available_type_profiles(cls) -> Mapping[str, Mapping[str, Any]]:
result = {}
for ft in SUPPORTED_FILE_TYPES:
result[f"{ft}_file_bundle"] = {"content_type": ft}
return result
serialize(self, data)
¶Source code in kiara/data_types/included_core_types/filesystem.py
def serialize(self, data: FileBundle) -> "SerializedData":
file_data: Dict[str, Any] = {}
file_metadata = {}
for rel_path, file in data.included_files.items():
file_data[rel_path] = {"type": "file", "codec": "raw", "file": file.path}
file_metadata[rel_path] = {
"file_name": file.file_name,
# "import_time": file.import_time,
}
metadata: Dict[str, Any] = {
"included_files": file_metadata,
"bundle_name": data.bundle_name,
# "import_time": data.import_time,
"size": data.size,
"number_of_files": data.number_of_files,
}
assert "__file_metadata__" not in file_data
file_data["__file_metadata__"] = {
"type": "inline-json",
"codec": "json",
"inline_data": metadata,
}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": file_data,
"serialization_profile": "copy",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_type": "deserialize.file_bundle",
"module_config": {
"value_type": "file_bundle",
"target_profile": "python_object",
"serialization_profile": "copy",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
FileTypeConfig (DataTypeConfig)
pydantic-model
¶
FileValueType (KiaraModelValueType)
¶A file.
Source code in kiara/data_types/included_core_types/filesystem.py
class FileValueType(KiaraModelValueType[FileModel, FileTypeConfig]):
"""A file."""
_data_type_name = "file"
@classmethod
def retrieve_available_type_profiles(cls) -> Mapping[str, Mapping[str, Any]]:
result = {}
for ft in SUPPORTED_FILE_TYPES:
result[f"{ft}_file"] = {"content_type": ft}
return result
@classmethod
def python_class(cls) -> Type:
return FileModel
@classmethod
def data_type_config_class(cls) -> Type[FileTypeConfig]:
return FileTypeConfig
def serialize(self, data: FileModel) -> "SerializedData":
_data = {
data.file_name: {
"type": "file",
"codec": "raw",
"file": data.path,
},
"__file_metadata__": {
"type": "inline-json",
"codec": "json",
"inline_data": {
"file_name": data.file_name,
# "import_time": data.import_time,
},
},
}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "copy",
"metadata": {
# "profile": "",
"environment": {},
"deserialize": {
"python_object": {
"module_type": "deserialize.file",
"module_config": {
"value_type": "file",
"target_profile": "python_object",
"serialization_profile": "copy",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
def create_model_from_python_obj(self, data: Any) -> FileModel:
if isinstance(data, Mapping):
return FileModel(**data)
if isinstance(data, str):
return FileModel.load_file(source=data)
else:
raise Exception(f"Can't create FileModel from data of type '{type(data)}'.")
def _pretty_print_as__string(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
data: Any = value.data
max_lines = render_config.get("max_lines", 34)
try:
lines = []
with open(data.path, "r", encoding="utf-8") as f:
for idx, l in enumerate(f):
if idx > max_lines:
lines.append("...\n")
lines.append("...")
break
lines.append(l)
# TODO: syntax highlighting
return "\n".join(lines)
except UnicodeDecodeError:
# found non-text data
lines = [
"Binary file or non-utf8 enconding, not printing content...",
"",
"[b]File metadata:[/b]",
"",
data.json(option=orjson.OPT_INDENT_2),
]
return "\n".join("lines")
def _pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
data: FileModel = value.data
max_lines = render_config.get("max_lines", 34)
try:
lines = []
with open(data.path, "r", encoding="utf-8") as f:
for idx, l in enumerate(f):
if idx > max_lines:
lines.append("...\n")
lines.append("...")
break
lines.append(l.rstrip())
return Group(*lines)
except UnicodeDecodeError:
# found non-text data
lines = [
"Binary file or non-utf8 enconding, not printing content...",
"",
"[b]File metadata:[/b]",
"",
data.json(option=orjson.OPT_INDENT_2),
]
return Group(*lines)
create_model_from_python_obj(self, data)
¶Source code in kiara/data_types/included_core_types/filesystem.py
def create_model_from_python_obj(self, data: Any) -> FileModel:
if isinstance(data, Mapping):
return FileModel(**data)
if isinstance(data, str):
return FileModel.load_file(source=data)
else:
raise Exception(f"Can't create FileModel from data of type '{type(data)}'.")
data_type_config_class()
classmethod
¶Source code in kiara/data_types/included_core_types/filesystem.py
@classmethod
def data_type_config_class(cls) -> Type[FileTypeConfig]:
return FileTypeConfig
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/filesystem.py
@classmethod
def python_class(cls) -> Type:
return FileModel
retrieve_available_type_profiles()
classmethod
¶Source code in kiara/data_types/included_core_types/filesystem.py
@classmethod
def retrieve_available_type_profiles(cls) -> Mapping[str, Mapping[str, Any]]:
result = {}
for ft in SUPPORTED_FILE_TYPES:
result[f"{ft}_file"] = {"content_type": ft}
return result
serialize(self, data)
¶Source code in kiara/data_types/included_core_types/filesystem.py
def serialize(self, data: FileModel) -> "SerializedData":
_data = {
data.file_name: {
"type": "file",
"codec": "raw",
"file": data.path,
},
"__file_metadata__": {
"type": "inline-json",
"codec": "json",
"inline_data": {
"file_name": data.file_name,
# "import_time": data.import_time,
},
},
}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "copy",
"metadata": {
# "profile": "",
"environment": {},
"deserialize": {
"python_object": {
"module_type": "deserialize.file",
"module_config": {
"value_type": "file",
"target_profile": "python_object",
"serialization_profile": "copy",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
internal
special
¶logger
¶
DocumentationModelValueType (InternalModelValueType)
¶Documentation for an internal entity.
Source code in kiara/data_types/included_core_types/internal/__init__.py
class DocumentationModelValueType(InternalModelValueType):
"""Documentation for an internal entity."""
_data_type_name = "doc"
def parse_python_obj(self, data: Any) -> DocumentationMetadataModel:
return DocumentationMetadataModel.create(data)
@classmethod
def python_class(cls) -> Type:
return DocumentationMetadataModel
def _pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
):
json_str = value.data.json(option=orjson.OPT_INDENT_2)
return Syntax(json_str, "json", background_color="default")
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
DocumentationMetadataModel |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/internal/__init__.py
def parse_python_obj(self, data: Any) -> DocumentationMetadataModel:
return DocumentationMetadataModel.create(data)
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
@classmethod
def python_class(cls) -> Type:
return DocumentationMetadataModel
InternalModelTypeConfig (DataTypeConfig)
pydantic-model
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
class InternalModelTypeConfig(DataTypeConfig):
kiara_model_id: Union[str, None] = Field(
description="The Python class backing this model (must sub-class 'KiaraModel')."
)
InternalModelValueType (InternalType)
¶A value type that is used internally.
This type should not be used by user-facing modules and/or operations.
Source code in kiara/data_types/included_core_types/internal/__init__.py
class InternalModelValueType(InternalType[KiaraModel, InternalModelTypeConfig]):
"""A value type that is used internally.
This type should not be used by user-facing modules and/or operations.
"""
_data_type_name = "internal_model"
_cls_cache: Union[Type[KiaraModel], None] = PrivateAttr(default=None)
@classmethod
def data_type_config_class(cls) -> Type[InternalModelTypeConfig]:
return InternalModelTypeConfig # type: ignore
def serialize(self, data: KiaraModel) -> Union[str, SerializedData]:
if self.type_config.kiara_model_id is None:
logger.debug(
"ignore.serialize_request",
data_type="internal_model",
cls=data.__class__.__name__,
reason="no model id in module config",
)
return NO_SERIALIZATION_MARKER
_data = {
"data": {
"type": "inline-json",
"inline_data": data.dict(),
"codec": "json",
},
}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "json",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_type": "load.internal_model",
"module_config": {
"value_type": "internal_model",
"target_profile": "python_object",
"serialization_profile": "json",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
@classmethod
def python_class(cls) -> Type:
return KiaraModel
@property
def model_cls(self) -> Type[KiaraModel]:
if self._cls_cache is not None:
return self._cls_cache
model_type_id = self.type_config.kiara_model_id
assert model_type_id is not None
model_registry = ModelRegistry.instance()
model_cls = model_registry.get_model_cls(
model_type_id, required_subclass=KiaraModel
)
self._cls_cache = model_cls
return self._cls_cache
def parse_python_obj(self, data: Any) -> KiaraModel:
if isinstance(data, KiaraModel):
return data
elif isinstance(data, Mapping):
return self.model_cls(**data)
else:
raise ValueError(
f"Can't parse data, invalid type '{type(data)}': must be subclass of 'KiaraModel' or Mapping."
)
def _validate(self, value: KiaraModel) -> None:
if not isinstance(value, KiaraModel):
raise Exception(f"Invalid type: {type(value)}.")
def _pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
):
json_str = value.data.json(option=orjson.OPT_INDENT_2)
return Syntax(json_str, "json", background_color="default")
model_cls: Type[kiara.models.KiaraModel]
property
readonly
¶data_type_config_class()
classmethod
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
@classmethod
def data_type_config_class(cls) -> Type[InternalModelTypeConfig]:
return InternalModelTypeConfig # type: ignore
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
KiaraModel |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/internal/__init__.py
def parse_python_obj(self, data: Any) -> KiaraModel:
if isinstance(data, KiaraModel):
return data
elif isinstance(data, Mapping):
return self.model_cls(**data)
else:
raise ValueError(
f"Can't parse data, invalid type '{type(data)}': must be subclass of 'KiaraModel' or Mapping."
)
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
@classmethod
def python_class(cls) -> Type:
return KiaraModel
serialize(self, data)
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
def serialize(self, data: KiaraModel) -> Union[str, SerializedData]:
if self.type_config.kiara_model_id is None:
logger.debug(
"ignore.serialize_request",
data_type="internal_model",
cls=data.__class__.__name__,
reason="no model id in module config",
)
return NO_SERIALIZATION_MARKER
_data = {
"data": {
"type": "inline-json",
"inline_data": data.dict(),
"codec": "json",
},
}
serialized_data = {
"data_type": self.data_type_name,
"data_type_config": self.type_config.dict(),
"data": _data,
"serialization_profile": "json",
"metadata": {
"environment": {},
"deserialize": {
"python_object": {
"module_type": "load.internal_model",
"module_config": {
"value_type": "internal_model",
"target_profile": "python_object",
"serialization_profile": "json",
},
}
},
},
}
from kiara.models.values.value import SerializationResult
serialized = SerializationResult(**serialized_data)
return serialized
InternalType (DataType, Generic)
¶'A 'marker' base data type for data types that are (mainly) used internally in kiara..
Source code in kiara/data_types/included_core_types/internal/__init__.py
class InternalType(
DataType[TYPE_PYTHON_CLS, TYPE_CONFIG_CLS],
Generic[TYPE_PYTHON_CLS, TYPE_CONFIG_CLS],
):
"""'A 'marker' base data type for data types that are (mainly) used internally in kiara.."""
_data_type_name = "internal"
@classmethod
def python_class(cls) -> Type:
return object
def pretty_print_as__string(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
if hasattr(self, "_pretty_print_as__string"):
return self._pretty_print_as_string(value=value, render_config=render_config) # type: ignore
return str(value.data)
def pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
):
if hasattr(self, "_pretty_print_as__terminal_renderable"):
return self._pretty_print_as__terminal_renderable(value=value, render_config=render_config) # type: ignore
data = value.data
from pydantic import BaseModel
if isinstance(data, BaseModel):
from kiara.utils.output import create_table_from_model_object
rendered = create_table_from_model_object(
model=data, render_config=render_config
)
elif isinstance(data, Iterable):
import pprint
rendered = pprint.pformat(data)
else:
rendered = str(data)
return rendered
def render_as__string(
self, value: "Value", render_config: "RenderScene", manifest: "Manifest"
):
if hasattr(self, "_render_as__string"):
return self._render_as__string(value=value, render_config=render_config, manifest=manifest) # type: ignore
else:
return self.pretty_print_as__string(value=value, render_config={})
def render_as__terminal_renderable(
self, value: "Value", render_config: "RenderScene", manifest: "Manifest"
):
if hasattr(self, "_render_as__terminal_renderable"):
return self._render_as__terminal(value=value, render_config=render_config, manifest=manifest) # type: ignore
return self.render_as__string(
value=value, render_config=render_config, manifest=manifest
)
pretty_print_as__string(self, value, render_config)
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
def pretty_print_as__string(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
if hasattr(self, "_pretty_print_as__string"):
return self._pretty_print_as_string(value=value, render_config=render_config) # type: ignore
return str(value.data)
pretty_print_as__terminal_renderable(self, value, render_config)
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
def pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
):
if hasattr(self, "_pretty_print_as__terminal_renderable"):
return self._pretty_print_as__terminal_renderable(value=value, render_config=render_config) # type: ignore
data = value.data
from pydantic import BaseModel
if isinstance(data, BaseModel):
from kiara.utils.output import create_table_from_model_object
rendered = create_table_from_model_object(
model=data, render_config=render_config
)
elif isinstance(data, Iterable):
import pprint
rendered = pprint.pformat(data)
else:
rendered = str(data)
return rendered
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
@classmethod
def python_class(cls) -> Type:
return object
render_as__string(self, value, render_config, manifest)
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
def render_as__string(
self, value: "Value", render_config: "RenderScene", manifest: "Manifest"
):
if hasattr(self, "_render_as__string"):
return self._render_as__string(value=value, render_config=render_config, manifest=manifest) # type: ignore
else:
return self.pretty_print_as__string(value=value, render_config={})
render_as__terminal_renderable(self, value, render_config, manifest)
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
def render_as__terminal_renderable(
self, value: "Value", render_config: "RenderScene", manifest: "Manifest"
):
if hasattr(self, "_render_as__terminal_renderable"):
return self._render_as__terminal(value=value, render_config=render_config, manifest=manifest) # type: ignore
return self.render_as__string(
value=value, render_config=render_config, manifest=manifest
)
TerminalRenderable (InternalType)
¶A list of renderable objects, used in the 'rich' Python library, to print to the terminal or in Jupyter.
Internally, the result list items can be either a string, a 'rich.console.ConsoleRenderable', or a 'rich.console.RichCast'.
Source code in kiara/data_types/included_core_types/internal/__init__.py
class TerminalRenderable(InternalType[object, DataTypeConfig]):
"""A list of renderable objects, used in the 'rich' Python library, to print to the terminal or in Jupyter.
Internally, the result list items can be either a string, a 'rich.console.ConsoleRenderable', or a 'rich.console.RichCast'.
"""
_data_type_name = "terminal_renderable"
@classmethod
def python_class(cls) -> Type:
return object
def _pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
renderable = value.data
table = Table(show_header=False, show_lines=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
cls = PythonClass.from_class(renderable.__class__)
table.add_row("python class", cls)
table.add_row("preview", Panel(renderable, height=20))
return table
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/internal/__init__.py
@classmethod
def python_class(cls) -> Type:
return object
render_value
¶
RenderSceneDataType (InternalType)
¶A value type to contain information about how to render a value in a specific render scenario.
Source code in kiara/data_types/included_core_types/internal/render_value.py
class RenderSceneDataType(InternalType[RenderScene, RenderSceneTypeConfig]):
"""A value type to contain information about how to render a value in a specific render scenario."""
_data_type_name = "render_scene"
def __init__(self, **type_config: Any):
self._cls_cache: Union[Type[RenderScene], None] = None
super().__init__(**type_config)
@classmethod
def python_class(cls) -> Type:
return RenderScene
@classmethod
def data_type_config_class(cls) -> Type[RenderSceneTypeConfig]:
return RenderSceneTypeConfig
@property
def model_cls(self) -> Type[RenderScene]:
if self._cls_cache is not None:
return self._cls_cache
kiara_model_id = self.type_config.kiara_model_id
if not kiara_model_id:
kiara_model_id = RenderScene._kiara_model_id
if kiara_model_id == RenderScene._kiara_model_id:
model_cls = RenderScene
else:
all_models = find_all_kiara_model_classes()
if kiara_model_id not in all_models.keys():
raise Exception(f"Invalid model id: {kiara_model_id}")
# TODO: check type is right?
model_cls = all_models[kiara_model_id] # type: ignore
assert issubclass(model_cls, RenderScene)
self._cls_cache = model_cls
return self._cls_cache
def parse_python_obj(self, data: Any) -> RenderScene:
if isinstance(data, RenderScene):
return data
elif isinstance(data, Mapping):
return self.model_cls(**data)
else:
raise ValueError(
f"Can't parse data, invalid type '{type(data)}': must be subclass of 'KiaraModel' or Mapping."
)
def _validate(self, value: RenderScene) -> None:
if not isinstance(value, RenderScene):
raise Exception(f"Invalid type: {type(value)}.")
def _pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
data: RenderScene = value.data
ri_json = data.json(option=orjson.orjson.OPT_INDENT_2)
return Syntax(ri_json, "json", background_color="default")
model_cls: Type[kiara.models.rendering.RenderScene]
property
readonly
¶data_type_config_class()
classmethod
¶Source code in kiara/data_types/included_core_types/internal/render_value.py
@classmethod
def data_type_config_class(cls) -> Type[RenderSceneTypeConfig]:
return RenderSceneTypeConfig
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
RenderScene |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/internal/render_value.py
def parse_python_obj(self, data: Any) -> RenderScene:
if isinstance(data, RenderScene):
return data
elif isinstance(data, Mapping):
return self.model_cls(**data)
else:
raise ValueError(
f"Can't parse data, invalid type '{type(data)}': must be subclass of 'KiaraModel' or Mapping."
)
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/internal/render_value.py
@classmethod
def python_class(cls) -> Type:
return RenderScene
RenderSceneTypeConfig (DataTypeConfig)
pydantic-model
¶Source code in kiara/data_types/included_core_types/internal/render_value.py
class RenderSceneTypeConfig(DataTypeConfig):
kiara_model_id: Union[str, None] = Field(
description="The id of the model backing this render (Python class must sub-class 'RenderScene').",
default=None,
)
RenderValueResultDataType (InternalType)
¶A value type to contain information about how to render a value in a specific render scenario.
Source code in kiara/data_types/included_core_types/internal/render_value.py
class RenderValueResultDataType(InternalType[RenderValueResult, DataTypeConfig]):
"""A value type to contain information about how to render a value in a specific render scenario."""
_data_type_name = "render_value_result"
def __init__(self, **type_config: Any):
self._cls_cache: Union[Type[RenderValueResult], None] = None
super().__init__(**type_config)
@classmethod
def python_class(cls) -> Type:
return RenderValueResult
def parse_python_obj(self, data: Any) -> RenderValueResult:
if data is None:
raise ValueError(
"Can't parse render_scene_result data: no source data provided (None)."
)
elif isinstance(data, RenderValueResult):
return data
elif isinstance(data, Mapping):
return RenderValueResult(**data)
else:
raise ValueError(
f"Can't parse data, invalid type '{type(data)}': must be subclass of 'RenderValueResult' or Mapping."
)
def _validate(self, value: Any) -> None:
if not isinstance(value, RenderValueResult):
raise Exception(f"Invalid type: {type(value)}.")
def _pretty_print_as__terminal_renderable(
self, value: "Value", render_config: Mapping[str, Any]
) -> Any:
data: RenderValueResult = value.data
ri_json = data.json(option=orjson.orjson.OPT_INDENT_2, exclude={"rendered"})
rendered = extract_renderable(data.rendered)
metadata = Syntax(ri_json, "json", background_color="default")
table = Table(show_header=True, box=box.SIMPLE)
table.add_column("Rendered item")
table.add_column("Render metadata")
table.add_row(rendered, metadata)
return table
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
RenderValueResult |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/internal/render_value.py
def parse_python_obj(self, data: Any) -> RenderValueResult:
if data is None:
raise ValueError(
"Can't parse render_scene_result data: no source data provided (None)."
)
elif isinstance(data, RenderValueResult):
return data
elif isinstance(data, Mapping):
return RenderValueResult(**data)
else:
raise ValueError(
f"Can't parse data, invalid type '{type(data)}': must be subclass of 'RenderValueResult' or Mapping."
)
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/internal/render_value.py
@classmethod
def python_class(cls) -> Type:
return RenderValueResult
serialization
¶
PythonObjectType (InternalType)
¶A 'plain' Python object.
This data type is mostly used internally, for hading over data in (de-)serialization operations.
Source code in kiara/data_types/included_core_types/serialization.py
class PythonObjectType(InternalType[object, DataTypeConfig]):
"""A 'plain' Python object.
This data type is mostly used internally, for hading over data in (de-)serialization operations.
"""
@classmethod
def python_class(cls) -> Type:
return object
def parse_python_obj(self, data: Any) -> object:
return data
def calculate_hash(self, data: SerializedData) -> str:
"""Calculate the hash of the value."""
return INVALID_HASH_MARKER
def calculate_size(self, data: SerializedData) -> int:
return INVALID_SIZE_MARKER
def _pretty_print_as__terminal_renderable(
self, value: Value, render_config: Mapping[str, Any]
):
return str(value.data)
calculate_hash(self, data)
¶Calculate the hash of the value.
Source code in kiara/data_types/included_core_types/serialization.py
def calculate_hash(self, data: SerializedData) -> str:
"""Calculate the hash of the value."""
return INVALID_HASH_MARKER
calculate_size(self, data)
¶Calculate the size of the value.
Source code in kiara/data_types/included_core_types/serialization.py
def calculate_size(self, data: SerializedData) -> int:
return INVALID_SIZE_MARKER
parse_python_obj(self, data)
¶Parse a value into a supported python type.
This exists to make it easier to do trivial conversions (e.g. from a date string to a datetime object). If you choose to overwrite this method, make 100% sure that you don't change the meaning of the value, and try to avoid adding or removing information from the data (e.g. by changing the resolution of a date).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
v |
the value |
required |
Returns:
| Type | Description |
|---|---|
object |
'None', if no parsing was done and the original value should be used, otherwise return the parsed Python object |
Source code in kiara/data_types/included_core_types/serialization.py
def parse_python_obj(self, data: Any) -> object:
return data
python_class()
classmethod
¶Source code in kiara/data_types/included_core_types/serialization.py
@classmethod
def python_class(cls) -> Type:
return object
defaults
¶
Attributes¶
ANY_TYPE_NAME
¶
ARRAY_MODEL_CATEOGORY_ID
¶
AUTHORS_METADATA_CATEGORY_ID
¶
BATCH_CONFIG_TYPE_CATEGORY_ID
¶
COLOR_LIST
¶
CONTEXT_INFO_CATEGORY_ID
¶
CONTEXT_METADATA_CATEOGORY_ID
¶
DATA_TYPES_CATEGORY_ID
¶
DATA_TYPE_CATEGORY_ID
¶
DATA_TYPE_CLASS_CATEGORY_ID
¶
DATA_WRAP_CATEGORY_ID
¶
DEFAULT_ALIAS_STORE_MARKER
¶
Name for the default context alias store.
DEFAULT_CONTEXT_NAME
¶
DEFAULT_DATA_STORE_MARKER
¶
Name for the default context data store.
DEFAULT_ENV_HASH_KEY
¶
DEFAULT_EXCLUDE_DIRS
¶
List of directory names to exclude by default when walking a folder recursively.
DEFAULT_EXCLUDE_FILES
¶
List of file names to exclude by default when reading folders.
DEFAULT_JOB_STORE_MARKER
¶
Name for the default context job store.
DEFAULT_NO_DESC_VALUE
¶
DEFAULT_PIPELINE_PARENT_ID
¶
Default parent id for pipeline objects that are not associated with a workflow.
DEFAULT_PRETTY_PRINT_CONFIG
¶
DEFAULT_TO_JSON_CONFIG: Mapping[str, Any]
¶
DEFAULT_WORKFLOW_STORE_MARKER
¶
Name for the default context workflow store.
DESTINY_CATEGORY_ID
¶
DOCUMENTATION_CATEGORY_ID
¶
ENVIRONMENT_TYPE_CATEGORY_ID
¶
FILE_BUNDLE_MODEL_CATEOGORY_ID
¶
FILE_MODEL_CATEOGORY_ID
¶
INVALID_HASH_MARKER
¶
INVALID_SIZE_MARKER
¶
INVALID_VALUE_NAMES
¶
List of reserved names, inputs/outputs can't use those.
JOB_CATEGORY_ID
¶
JOB_CONFIG_TYPE_CATEGORY_ID
¶
JOB_LOG_CATEGORY_ID
¶
JOB_RECORD_TYPE_CATEGORY_ID
¶
KIARA_CONFIG_FILE_NAME
¶
KIARA_DB_MIGRATIONS_CONFIG
¶
KIARA_DB_MIGRATIONS_FOLDER
¶
KIARA_DEFAULT_ROOT_NODE_ID
¶
KIARA_DEV_CONFIG_FILE
¶
KIARA_DEV_CONFIG_FILE_NAME
¶
KIARA_HASH_FUNCTION
¶
KIARA_MAIN_CONFIG_FILE
¶
KIARA_MAIN_CONTEXTS_PATH
¶
KIARA_MODULE_BASE_FOLDER
¶
Marker to indicate the base folder for the kiara module.
KIARA_MODULE_METADATA_ATTRIBUTE
¶
KIARA_RESOURCES_FOLDER
¶
Default resources folder for this package.
KIARA_ROOT_TYPE_NAME
¶
LOAD_CONFIG_DATA_TYPE_NAME
¶
LOAD_CONFIG_PLACEHOLDER
¶
METADATA_DESTINY_STORE_MARKER
¶
Name for the default context destiny store.
MODULE_CONFIG_CATEGORY_ID
¶
MODULE_CONFIG_METADATA_CATEGORY_ID
¶
MODULE_CONFIG_SCHEMA_CATEGORY_ID
¶
MODULE_TYPES_CATEGORY_ID
¶
MODULE_TYPE_CATEGORY_ID
¶
MODULE_TYPE_KEY
¶
The key to specify the type of a module.
MODULE_TYPE_NAME_KEY
¶
The string for the module type name in a module configuration dict.
NONE_STORE_ID
¶
NONE_VALUE_ID
¶
NOT_SET_VALUE_ID
¶
NO_HASH_MARKER
¶
Marker string to indicate no hash was calculated.
NO_MODULE_TYPE
¶
NO_SERIALIZATION_MARKER
¶
NO_VALUE_ID_MARKER
¶
Marker string to indicate no value id exists.
OPERATIONS_CATEGORY_ID
¶
OPERATION_CATEOGORY_ID
¶
OPERATION_CONFIG_CATEOGORY_ID
¶
OPERATION_DETAILS_CATEOGORY_ID
¶
OPERATION_INPUTS_SCHEMA_CATEOGORY_ID
¶
OPERATION_OUTPUTS_SCHEMA_CATEOGORY_ID
¶
OPERATION_TYPES_CATEGORY_ID
¶
OPERATION_TYPE_CATEGORY_ID
¶
ORPHAN_PEDIGREE_OUTPUT_NAME
¶
PIPELINE_CONFIG_TYPE_CATEGORY_ID
¶
PIPELINE_PARENT_MARKER
¶
Marker string in the pipeline structure that indicates a parent pipeline element.
PIPELINE_STEP_DETAILS_CATEGORY_ID
¶
PIPELINE_STEP_TYPE_CATEGORY_ID
¶
PIPELINE_STRUCTURE_TYPE_CATEGORY_ID
¶
PIPELINE_TYPES_CATEGORY_ID
¶
PIPELINE_TYPE_CATEGORY_ID
¶
PYDANTIC_USE_CONSTRUCT: bool
¶
SERIALIZED_DATA_TYPE_NAME
¶
STEP_ID_KEY
¶
The key to specify the step id.
STRICT_CHECKS: bool
¶
TABLE_MODEL_CATEOGORY_ID
¶
UNOLOADABLE_DATA_CATEGORY_ID
¶
USER_PIPELINES_FOLDER
¶
VALID_PIPELINE_FILE_EXTENSIONS
¶
File extensions a kiara pipeline/workflow file can have.
VALUES_CATEGORY_ID
¶
VALUE_CATEGORY_ID
¶
VALUE_METADATA_CATEGORY_ID
¶
VALUE_PEDIGREE_TYPE_CATEGORY_ID
¶
VALUE_SCHEMA_CATEGORY_ID
¶
VOID_KIARA_ID
¶
kiara_app_dirs
¶
Classes¶
doc
special
¶
Main module for code that helps with documentation auto-generation in supported projects.
Classes¶
FrklDocumentationPlugin (BasePlugin)
¶
mkdocs plugin to render API documentation for a project.
To add to a project, add this to the 'plugins' section of a mkdocs config file:
- frkl-docgen:
main_module: "module_name"
This will add an API reference navigation item to your page navigation, with auto-generated entries for every
Python module in your package.
Source code in kiara/doc/__init__.py
class FrklDocumentationPlugin(BasePlugin):
"""[mkdocs](https://www.mkdocs.org/) plugin to render API documentation for a project.
To add to a project, add this to the 'plugins' section of a mkdocs config file:
```yaml
- frkl-docgen:
main_module: "module_name"
```
This will add an ``API reference`` navigation item to your page navigation, with auto-generated entries for every
Python module in your package.
"""
config_scheme = (("main_module", mkdocs.config.config_options.Type(str)),)
def __init__(self):
self._doc_paths = None
self._dir = tempfile.TemporaryDirectory(prefix="frkl_doc_gen_")
self._doc_files = None
super().__init__()
def on_files(self, files: Files, config: Config) -> Files:
self._doc_paths = gen_pages_for_module(self.config["main_module"])
self._doc_files = {}
for k in sorted(self._doc_paths, key=lambda x: os.path.splitext(x)[0]):
content = self._doc_paths[k]["content"]
_file = File(
k,
src_dir=self._dir.name,
dest_dir=config["site_dir"],
use_directory_urls=config["use_directory_urls"],
)
os.makedirs(os.path.dirname(_file.abs_src_path), exist_ok=True)
with open(_file.abs_src_path, "w") as f:
f.write(content)
self._doc_files[k] = _file
files.append(_file)
return files
def on_page_content(self, html, page: Page, config: Config, files: Files):
repo_url = config.get("repo_url", None)
python_src = config.get("edit_uri", None)
if page.file.src_path in self._doc_paths.keys():
src_path = self._doc_paths.get(page.file.src_path)["python_src"]["rel_path"]
rel_base = urllib.parse.urljoin(repo_url, f"{python_src}/../src/{src_path}")
page.edit_url = rel_base
return html
def on_nav(self, nav: Navigation, config: Config, files: Files):
for item in nav.items:
if item.title and "Api reference" in item.title:
return nav
pages = []
for _file in self._doc_files.values():
pages.append(_file.page)
section = Section(title="API reference", children=pages)
nav.items.append(section)
nav.pages.extend(pages)
_add_previous_and_next_links(nav.pages)
_add_parent_links(nav.items)
return nav
def on_post_build(self, config: Config):
self._dir.cleanup()
config_scheme
¶on_files(self, files, config)
¶Source code in kiara/doc/__init__.py
def on_files(self, files: Files, config: Config) -> Files:
self._doc_paths = gen_pages_for_module(self.config["main_module"])
self._doc_files = {}
for k in sorted(self._doc_paths, key=lambda x: os.path.splitext(x)[0]):
content = self._doc_paths[k]["content"]
_file = File(
k,
src_dir=self._dir.name,
dest_dir=config["site_dir"],
use_directory_urls=config["use_directory_urls"],
)
os.makedirs(os.path.dirname(_file.abs_src_path), exist_ok=True)
with open(_file.abs_src_path, "w") as f:
f.write(content)
self._doc_files[k] = _file
files.append(_file)
return files
on_nav(self, nav, config, files)
¶Source code in kiara/doc/__init__.py
def on_nav(self, nav: Navigation, config: Config, files: Files):
for item in nav.items:
if item.title and "Api reference" in item.title:
return nav
pages = []
for _file in self._doc_files.values():
pages.append(_file.page)
section = Section(title="API reference", children=pages)
nav.items.append(section)
nav.pages.extend(pages)
_add_previous_and_next_links(nav.pages)
_add_parent_links(nav.items)
return nav
on_page_content(self, html, page, config, files)
¶Source code in kiara/doc/__init__.py
def on_page_content(self, html, page: Page, config: Config, files: Files):
repo_url = config.get("repo_url", None)
python_src = config.get("edit_uri", None)
if page.file.src_path in self._doc_paths.keys():
src_path = self._doc_paths.get(page.file.src_path)["python_src"]["rel_path"]
rel_base = urllib.parse.urljoin(repo_url, f"{python_src}/../src/{src_path}")
page.edit_url = rel_base
return html
on_post_build(self, config)
¶Source code in kiara/doc/__init__.py
def on_post_build(self, config: Config):
self._dir.cleanup()
Modules¶
gen_info_pages
¶
generate_detail_pages(context_info, sub_path='info', add_summary_page=False)
¶Source code in kiara/doc/gen_info_pages.py
def generate_detail_pages(
context_info: KiaraContextInfo,
sub_path: str = "info",
add_summary_page: bool = False,
):
pages = {}
summary = []
all_info = context_info.get_all_info(skip_empty_types=True)
for item_type, items_info in all_info.items():
summary.append(f"* [{item_type}]({item_type}.md)")
path = render_item_listing(
item_type=item_type, items=items_info.item_infos, sub_path=sub_path
)
pages[item_type] = path
if summary:
if add_summary_page:
summary.insert(0, "* [Summary](index.md)")
with mkdocs_gen_files.open(f"{sub_path}/SUMMARY.md", "w") as f:
f.write("\n".join(summary))
return pages
get_jina_env()
¶Source code in kiara/doc/gen_info_pages.py
def get_jina_env():
global _jinja_env
if _jinja_env is None:
from jinja2 import Environment, FileSystemLoader
_jinja_env = Environment(
loader=FileSystemLoader(
os.path.join(KIARA_RESOURCES_FOLDER, "templates", "doc_gen"),
encoding="utf8",
)
)
return _jinja_env
render_item_listing(item_type, items, sub_path='info')
¶Source code in kiara/doc/gen_info_pages.py
def render_item_listing(
item_type: str, items: Mapping[str, ItemInfo], sub_path: str = "info"
):
list_template = get_jina_env().get_template("info_listing.j2")
render_args = {"items": items, "item_type": item_type}
rendered = list_template.render(**render_args)
path = f"{sub_path}/{item_type}.md"
with mkdocs_gen_files.open(path, "w") as f:
f.write(rendered)
return path
generate_api_doc
¶
Functions¶
gen_pages_for_module(module, prefix='api_reference')
¶Generate modules for a set of modules (using the mkdocstring package.
Source code in kiara/doc/generate_api_doc.py
def gen_pages_for_module(
module: typing.Union[str, ModuleType], prefix: str = "api_reference"
):
"""Generate modules for a set of modules (using the [mkdocstring](https://github.com/mkdocstrings/mkdocstrings) package."""
result = {}
modules_info = get_source_tree(module)
for module_name, path in modules_info.items():
page_name = module_name
if page_name.endswith("__init__"):
page_name = page_name[0:-9]
if page_name.endswith("._frkl"):
continue
doc_path = f"{prefix}{os.path.sep}{page_name}.md"
p = Path("..", path["abs_path"])
if not p.read_text().strip():
continue
main_module = path["main_module"]
if page_name == main_module:
title = page_name
else:
title = page_name.replace(f"{main_module}.", "➜ ")
result[doc_path] = {
"python_src": path,
"content": f"---\ntitle: {title}\n---\n# {page_name}\n\n::: {module_name}",
}
return result
get_source_tree(module)
¶Find all python source files for a module.
Source code in kiara/doc/generate_api_doc.py
def get_source_tree(module: typing.Union[str, ModuleType]):
"""Find all python source files for a module."""
if isinstance(module, str):
module = importlib.import_module(module)
if not isinstance(module, ModuleType):
raise TypeError(
f"Invalid type '{type(module)}', input needs to be a string or module."
)
module_file = module.__file__
assert module_file is not None
module_root = os.path.dirname(module_file)
module_name = module.__name__
src = {}
for path in Path(module_root).glob("**/*.py"):
rel = os.path.relpath(path, module_root)
mod_name = f"{module_name}.{rel[0:-3].replace(os.path.sep, '.')}"
rel_path = f"{module_name}{os.path.sep}{rel}"
src[mod_name] = {
"rel_path": rel_path,
"abs_path": path,
"main_module": module_name,
}
return src
mkdocs_macros_cli
¶
KIARA_DOC_BUILD_CACHE_DIR
¶os_env_vars
¶Functions¶
define_env(env)
¶Helper macros for Python project documentation.
Currently, those macros are available (check the source code for more details):
cli¶
Execute a command on the command-line, capture the output and return it to be used in a documentation page.
inline_file_as_codeblock¶
Read an external file, and return its content as a markdown code block.
Source code in kiara/doc/mkdocs_macros_cli.py
def define_env(env):
"""
Helper macros for Python project documentation.
Currently, those macros are available (check the source code for more details):
## ``cli``
Execute a command on the command-line, capture the output and return it to be used in a documentation page.
## ``inline_file_as_codeblock``
Read an external file, and return its content as a markdown code block.
"""
# env.variables["baz"] = "John Doe"
@env.macro
def cli(
*command,
print_command: bool = True,
code_block: bool = True,
split_command_and_output: bool = True,
max_height: Union[int, None] = None,
cache_key: Union[str, None] = None,
extra_env: Union[Dict[str, str], None] = None,
fake_command: Union[str, None] = None,
fail_ok: bool = False,
repl_dict: Union[Mapping[str, str], None] = None,
):
"""Execute the provided command, save the output and return it to be used in documentation modules."""
hashes = DeepHash(command)
hash_str = hashes[command]
hashes_env = DeepHash(extra_env)
hashes_env_str = hashes_env[extra_env]
hash_str = hash_str + "_" + hashes_env_str
if cache_key:
hash_str = hash_str + "_" + cache_key
cache_file: Path = Path(os.path.join(KIARA_DOC_BUILD_CACHE_DIR, str(hash_str)))
failed_cache_file: Path = Path(
os.path.join(KIARA_DOC_BUILD_CACHE_DIR, f"{hash_str}.failed")
)
cache_info_file: Path = Path(
os.path.join(KIARA_DOC_BUILD_CACHE_DIR), f"{hash_str}.command"
)
_run_env = dict(os_env_vars)
if extra_env:
_run_env.update(extra_env)
if cache_file.is_file():
stdout_str = cache_file.read_text()
if repl_dict:
for k, v in repl_dict.items():
stdout_str = stdout_str.replace(k, v)
else:
start = timer()
cache_info = {
"command": command,
"extra_env": extra_env,
"cmd_hash": hash_str,
"cache_key": cache_key,
"fail_ok": fail_ok,
"started": start,
"repl_dict": repl_dict,
}
print(f"RUNNING: {' '.join(command)}")
p = Popen(command, stdout=PIPE, stderr=PIPE, env=_run_env)
stdout, stderr = p.communicate()
stdout_str = stdout.decode("utf-8")
stderr_str = stderr.decode("utf-8")
if repl_dict:
for k, v in repl_dict.items():
stdout_str = stdout_str.replace(k, v)
stderr_str = stderr_str.replace(k, v)
print("stdout:")
print(stdout_str)
print("stderr:")
print(stderr_str)
cache_info["exit_code"] = p.returncode
end = timer()
if p.returncode == 0:
# result = subprocess.check_output(command, env=_run_env)
# stdout = result.decode()
cache_file.write_bytes(stdout)
cache_info["size"] = len(stdout)
cache_info["duration"] = end - start
cache_info["success"] = True
cache_info["output_file"] = cache_file.as_posix()
cache_info_file.write_bytes(orjson.dumps(cache_info))
if failed_cache_file.exists():
failed_cache_file.unlink()
else:
cache_info["duration"] = end - start
if fail_ok:
cache_info["size"] = len(stdout)
cache_info["success"] = True
cache_file.write_bytes(stdout)
cache_info["output_file"] = cache_file.as_posix()
cache_info_file.write_bytes(orjson.dumps(cache_info))
if failed_cache_file.exists():
failed_cache_file.unlink()
else:
cache_info["size"] = len(stdout)
cache_info["success"] = False
failed_cache_file.write_bytes(stdout)
cache_info["output_file"] = failed_cache_file.as_posix()
cache_info_file.write_bytes(orjson.dumps(cache_info))
# stdout = f"Error: {e}\n\nStdout: {e.stdout}\n\nStderr: {e.stderr}"
# cache_info["size"] = len(stdout)
# cache_info["success"] = False
# print("stdout:")
# print(e.stdout)
# print("stderr:")
# print(e.stderr)
# failed_cache_file.write_text(stdout)
# cache_info["output_file"] = failed_cache_file.as_posix()
# cache_info_file.write_bytes(orjson.dumps(cache_info))
if os.getenv("FAIL_DOC_BUILD_ON_ERROR") == "true":
sys.exit(1)
if fake_command:
command_str = fake_command
else:
command_str = " ".join(command)
if split_command_and_output and print_command:
_c = f"\n``` console\n{command_str}\n```\n"
_output = "``` console\n" + stdout_str + "\n```\n"
if max_height is not None and max_height > 0:
_output = f"<div style='max-height:{max_height}px;overflow:auto'>\n{_output}\n</div>"
_stdout = _c + _output
else:
if print_command:
_stdout = f"> {command_str}\n{stdout_str}"
if code_block:
_stdout = "``` console\n" + _stdout + "\n```\n"
if max_height is not None and max_height > 0:
_stdout = f"<div style='max-height:{max_height}px;overflow:auto'>\n{_stdout}\n</div>"
return _stdout
@env.macro
def inline_file_as_codeblock(path, format: str = ""):
"""Import external file and return its content as a markdown code block."""
f = Path(path)
return f"```{format}\n{f.read_text()}\n```"
mkdocs_macros_kiara
¶
kiara_obj
¶yaml
¶Functions¶
define_env(env)
¶This is the hook for defining variables, macros and filters
- variables: the dictionary that contains the environment variables
- macro: a decorator function, to declare a macro.
Source code in kiara/doc/mkdocs_macros_kiara.py
def define_env(env):
"""
This is the hook for defining variables, macros and filters
- variables: the dictionary that contains the environment variables
- macro: a decorator function, to declare a macro.
"""
# env.variables["baz"] = "John Doe"
@env.macro
def get_schema_for_model(model_class: Union[str, Type[BaseModel]]):
if isinstance(model_class, str):
_class: Type[BaseModel] = locate(model_class) # type: ignore
else:
_class = model_class
schema_json = _class.schema_json(indent=2)
return schema_json
@env.macro
def get_src_of_object(obj: Union[str, Any]):
try:
if isinstance(obj, str):
_obj: Type[BaseModel] = locate(obj) # type: ignore
else:
_obj = obj
src = inspect.getsource(_obj)
return src
except Exception as e:
return f"Can't render object source: {str(e)}"
@env.macro
def get_context_info() -> KiaraContextInfo:
return builtins.plugin_package_context_info # type: ignore
# @env.macro
# def get_module_info(module_type: str):
#
# try:
#
# m_cls = Kiara.instance().module_registry.get_module_class(module_type)
# info = KiaraModuleTypeInfo.from_module_class(m_cls)
#
# from rich.console import Console
#
# console = Console(record=True)
# console.print(info)
#
# html = console.export_text()
# return html
# except Exception as e:
# return f"Can't render module info: {str(e)}"
#
# @env.macro
# def get_info_item_list_for_category(
# category: str, limit_to_package: typing.Optional[str] = None
# ) -> typing.Dict[str, KiaraInfoModel]:
# return _get_info_item_list_for_category(
# category=category, limit_to_package=limit_to_package
# )
#
# def _get_info_item_list_for_category(
# category: str, limit_to_package: typing.Optional[str] = None
# ) -> typing.Dict[str, KiaraInfoModel]:
#
# infos = kiara_context.find_subcomponents(category=category)
#
# if limit_to_package:
# temp = {}
# for n_id, obj in infos.items():
# if obj.context.labels.get("package", None) == limit_to_package:
# temp[n_id] = obj
# infos = temp
#
# docs = {}
# for n_id, obj in infos.items():
# docs[obj.get_id()] = obj.documentation.description
#
# return docs
#
# @env.macro
# def get_info_for_categories(
# *categories: str, limit_to_package: typing.Optional[str] = None
# ):
#
# TITLE_MAP = {
# "metadata.module": "Modules",
# "metadata.pipeline": "Pipelines",
# "metadata.type": "Value data_types",
# "metadata.operation_type": "Operation data_types",
# }
# result = {}
# for cat in categories:
# infos = _get_info_item_list_for_category(
# cat, limit_to_package=limit_to_package
# )
# if infos:
# result[cat] = {"items": infos, "title": TITLE_MAP[cat]}
#
# return result
#
# @env.macro
# def get_module_list_for_package(
# package_name: str,
# include_core_modules: bool = True,
# include_pipelines: bool = True,
# ):
#
# modules = kiara_obj.module_registry.find_modules_for_package(
# package_name,
# include_core_modules=include_core_modules,
# include_pipelines=include_pipelines,
# )
#
# result = []
# for name, info in modules.items():
# type_md = info.get_type_metadata()
# result.append(
# f"[``{name}``][kiara_info.modules.{name}]: {type_md.documentation.description}"
# )
#
# return result
#
# @env.macro
# def get_data_types_for_package(package_name: str):
#
# data_types = kiara_obj.type_registry.find_data_type_classes_for_package(
# package_name
# )
# result = []
# for name, info in data_types.items():
# type_md = info.get_type_metadata()
# result.append(f" - ``{name}``: {type_md.documentation.description}")
#
# return "\n".join(result)
#
# @env.macro
# def get_metadata_models_for_package(package_name: str):
#
# metadata_schemas = kiara_obj.metadata_mgmt.find_all_models_for_package(
# package_name
# )
# result = []
# for name, info in metadata_schemas.items():
# type_md = info.get_type_metadata()
# result.append(f" - ``{name}``: {type_md.documentation.description}")
#
# return "\n".join(result)
#
# @env.macro
# def get_kiara_context() -> KiaraContext:
# return kiara_context
mkdocstrings
special
¶
Modules¶
collector
¶logger
¶
KiaraCollector (BaseCollector)
¶The class responsible for loading Jinja templates and rendering them.
It defines some configuration options, implements the render method,
and overrides the update_env method of the [BaseRenderer class][mkdocstrings.handlers.base.BaseRenderer].
Source code in kiara/doc/mkdocstrings/collector.py
class KiaraCollector(BaseCollector):
"""The class responsible for loading Jinja templates and rendering them.
It defines some configuration options, implements the `render` method,
and overrides the `update_env` method of the [`BaseRenderer` class][mkdocstrings.handlers.base.BaseRenderer].
"""
default_config: dict = {"docstring_style": "google", "docstring_options": {}}
"""The default selection options.
Option | Type | Description | Default
------ | ---- | ----------- | -------
**`docstring_style`** | `"google" | "numpy" | "sphinx" | None` | The docstring style to use. | `"google"`
**`docstring_options`** | `dict[str, Any]` | The options for the docstring parser. | `{}`
"""
fallback_config: dict = {"fallback": True}
def __init__(self) -> None:
"""Initialize the collector."""
self._kiara: Kiara = Kiara.instance()
def collect(self, identifier: str, config: dict) -> CollectorItem: # noqa: WPS231
"""Collect the documentation tree given an identifier and selection options.
Arguments:
identifier: The dotted-path of a Python object available in the Python path.
config: Selection options, used to alter the data collection done by `pytkdocs`.
Raises:
CollectionError: When there was a problem collecting the object documentation.
Returns:
The collected object-tree.
"""
tokens = identifier.split(".")
if tokens[0] != "kiara_info":
return None
item_type = tokens[1]
item_id = ".".join(tokens[2:])
if not item_id:
raise CollectionError(f"Invalid id: {identifier}")
ctx: KiaraContextInfo = builtins.plugin_package_context_info # type: ignore
try:
item: ItemInfo = ctx.get_info(item_type=item_type, item_id=item_id)
except Exception as e:
log_exception(e)
raise CollectionError(f"Invalid id: {identifier}")
return {"obj": item, "identifier": identifier}
default_config: dict
¶The default selection options.
Option | Type | Description | Default
------ | ---- | ----------- | -------
docstring_style | "google" | "numpy" | "sphinx" | None | The docstring style to use. | "google"
docstring_options | dict[str, Any] | The options for the docstring parser. | {}
fallback_config: dict
¶collect(self, identifier, config)
¶Collect the documentation tree given an identifier and selection options.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
identifier |
str |
The dotted-path of a Python object available in the Python path. |
required |
config |
dict |
Selection options, used to alter the data collection done by |
required |
Exceptions:
| Type | Description |
|---|---|
CollectionError |
When there was a problem collecting the object documentation. |
Returns:
| Type | Description |
|---|---|
CollectorItem |
The collected object-tree. |
Source code in kiara/doc/mkdocstrings/collector.py
def collect(self, identifier: str, config: dict) -> CollectorItem: # noqa: WPS231
"""Collect the documentation tree given an identifier and selection options.
Arguments:
identifier: The dotted-path of a Python object available in the Python path.
config: Selection options, used to alter the data collection done by `pytkdocs`.
Raises:
CollectionError: When there was a problem collecting the object documentation.
Returns:
The collected object-tree.
"""
tokens = identifier.split(".")
if tokens[0] != "kiara_info":
return None
item_type = tokens[1]
item_id = ".".join(tokens[2:])
if not item_id:
raise CollectionError(f"Invalid id: {identifier}")
ctx: KiaraContextInfo = builtins.plugin_package_context_info # type: ignore
try:
item: ItemInfo = ctx.get_info(item_type=item_type, item_id=item_id)
except Exception as e:
log_exception(e)
raise CollectionError(f"Invalid id: {identifier}")
return {"obj": item, "identifier": identifier}
handler
¶
KiaraHandler (BaseHandler)
¶The kiara handler class.
Attributes:
| Name | Type | Description |
|---|---|---|
domain |
str |
The cross-documentation domain/language for this handler. |
enable_inventory |
bool |
Whether this handler is interested in enabling the creation
of the |
Source code in kiara/doc/mkdocstrings/handler.py
class KiaraHandler(BaseHandler):
"""The kiara handler class.
Attributes:
domain: The cross-documentation domain/language for this handler.
enable_inventory: Whether this handler is interested in enabling the creation
of the `objects.inv` Sphinx inventory file.
"""
domain: str = "kiara"
enable_inventory: bool = True
# load_inventory = staticmethod(inventory.list_object_urls)
#
# @classmethod
# def load_inventory(
# cls,
# in_file: typing.BinaryIO,
# url: str,
# base_url: typing.Optional[str] = None,
# **kwargs: typing.Any,
# ) -> typing.Iterator[typing.Tuple[str, str]]:
# """Yield items and their URLs from an inventory file streamed from `in_file`.
# This implements mkdocstrings' `load_inventory` "protocol" (see plugin.py).
# Arguments:
# in_file: The binary file-like object to read the inventory from.
# url: The URL that this file is being streamed from (used to guess `base_url`).
# base_url: The URL that this inventory's sub-paths are relative to.
# **kwargs: Ignore additional arguments passed from the config.
# Yields:
# Tuples of (item identifier, item URL).
# """
#
# print("XXXXXXXXXXXXXXXXXXXXXXXXXXXX")
#
# if base_url is None:
# base_url = posixpath.dirname(url)
#
# for item in Inventory.parse_sphinx(
# in_file, domain_filter=("py",)
# ).values(): # noqa: WPS526
# yield item.name, posixpath.join(base_url, item.uri)
get_handler(theme, custom_templates=None, **config)
¶Simply return an instance of PythonHandler.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
theme |
str |
The theme to use when rendering contents. |
required |
custom_templates |
Optional[str] |
Directory containing custom templates. |
None |
**config |
Any |
Configuration passed to the handler. |
{} |
Returns:
| Type | Description |
|---|---|
KiaraHandler |
An instance of |
Source code in kiara/doc/mkdocstrings/handler.py
def get_handler(
theme: str, # noqa: W0613 (unused argument config)
custom_templates: typing.Union[str, None] = None,
**config: typing.Any,
) -> KiaraHandler:
"""Simply return an instance of `PythonHandler`.
Arguments:
theme: The theme to use when rendering contents.
custom_templates: Directory containing custom templates.
**config: Configuration passed to the handler.
Returns:
An instance of `PythonHandler`.
"""
if custom_templates is not None:
raise Exception("Custom templates are not supported for the kiara renderer.")
custom_templates = os.path.join(
KIARA_RESOURCES_FOLDER, "templates", "info_templates"
)
return KiaraHandler(
collector=KiaraCollector(),
renderer=KiaraInfoRenderer("kiara", theme, custom_templates),
)
renderer
¶
AliasResolutionError
¶Source code in kiara/doc/mkdocstrings/renderer.py
class AliasResolutionError:
pass
KiaraInfoRenderer (BaseRenderer)
¶Source code in kiara/doc/mkdocstrings/renderer.py
class KiaraInfoRenderer(BaseRenderer):
default_config: dict = {}
def get_anchors(
self, data: CollectorItem
) -> typing.List[str]: # noqa: D102 (ignore missing docstring)
if data is None:
return list()
return list([data["identifier"], data["kiara_id"], data["obj"].get_id()])
def render(self, data: typing.Dict[str, typing.Any], config: dict) -> str:
# final_config = ChainMap(config, self.default_config)
obj = data["obj"]
html = obj.create_html()
return html
default_config: dict
¶get_anchors(self, data)
¶Return the possible identifiers (HTML anchors) for a collected item.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data |
Any |
The collected data. |
required |
Returns:
| Type | Description |
|---|---|
List[str] |
The HTML anchors (without '#'), or an empty tuple if this item doesn't have an anchor. |
Source code in kiara/doc/mkdocstrings/renderer.py
def get_anchors(
self, data: CollectorItem
) -> typing.List[str]: # noqa: D102 (ignore missing docstring)
if data is None:
return list()
return list([data["identifier"], data["kiara_id"], data["obj"].get_id()])
render(self, data, config)
¶Render a template using provided data and configuration options.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data |
Dict[str, Any] |
The collected data to render. |
required |
config |
dict |
The rendering options. |
required |
Returns:
| Type | Description |
|---|---|
str |
The rendered template as HTML. |
Source code in kiara/doc/mkdocstrings/renderer.py
def render(self, data: typing.Dict[str, typing.Any], config: dict) -> str:
# final_config = ChainMap(config, self.default_config)
obj = data["obj"]
html = obj.create_html()
return html
exceptions
¶
FailedJobException (Exception)
¶
Source code in kiara/exceptions.py
class FailedJobException(Exception):
def __init__(self, job: "ActiveJob", msg: Union[str, None] = None):
self.job: ActiveJob = job
if msg is None:
msg = "Job failed."
self.msg = msg
super().__init__(msg)
def create_renderable(self, **config: Any):
from rich import box
from rich.console import Group
from rich.panel import Panel
from rich.table import Table
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
table.add_row("job_id", str(self.job.job_id))
table.add_row("module_type", self.job.job_config.module_type)
group = Group(
Panel(f"[red]Error[/red]: [i]{self.msg}[i]", box=box.SIMPLE), table
)
return group
create_renderable(self, **config)
¶Source code in kiara/exceptions.py
def create_renderable(self, **config: Any):
from rich import box
from rich.console import Group
from rich.panel import Panel
from rich.table import Table
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
table.add_row("job_id", str(self.job.job_id))
table.add_row("module_type", self.job.job_config.module_type)
group = Group(
Panel(f"[red]Error[/red]: [i]{self.msg}[i]", box=box.SIMPLE), table
)
return group
InvalidValuesException (Exception)
¶
Source code in kiara/exceptions.py
class InvalidValuesException(Exception):
def __init__(
self,
msg: Union[None, str, Exception] = None,
invalid_values: Mapping[str, str] = None,
):
if invalid_values is None:
invalid_values = {}
self.invalid_inputs: Mapping[str, str] = invalid_values
if msg is None:
if not self.invalid_inputs:
_msg = "Invalid values. No details available."
else:
msg_parts = []
for k, v in invalid_values.items():
msg_parts.append(f"{k}: {v}")
_msg = f"Invalid values: {', '.join(msg_parts)}"
elif isinstance(msg, Exception):
self._parent: Union[Exception, None] = msg
_msg = str(msg)
else:
self._parent = None
_msg = msg
super().__init__(_msg)
def create_renderable(self, **config: Any) -> "Table":
from rich import box
from rich.console import RenderableType
from rich.table import Table
table = Table(box=box.SIMPLE, show_header=True)
table.add_column("field name", style="i")
table.add_column("[red]error[/red]")
for field_name, error in self.invalid_inputs.items():
row: List[RenderableType] = [field_name]
row.append(error)
table.add_row(*row)
return table
create_renderable(self, **config)
¶Source code in kiara/exceptions.py
def create_renderable(self, **config: Any) -> "Table":
from rich import box
from rich.console import RenderableType
from rich.table import Table
table = Table(box=box.SIMPLE, show_header=True)
table.add_column("field name", style="i")
table.add_column("[red]error[/red]")
for field_name, error in self.invalid_inputs.items():
row: List[RenderableType] = [field_name]
row.append(error)
table.add_row(*row)
return table
JobConfigException (Exception)
¶
Source code in kiara/exceptions.py
class JobConfigException(Exception):
def __init__(
self,
msg: Union[str, Exception],
manifest: "Manifest",
inputs: Mapping[str, Any],
):
self._manifest: Manifest = manifest
self._inputs: Mapping[str, Any] = inputs
if isinstance(msg, Exception):
self._parent: Union[Exception, None] = msg
_msg = str(msg)
else:
self._parent = None
_msg = msg
super().__init__(_msg)
@property
def manifest(self) -> "Manifest":
return self._manifest
@property
def inputs(self) -> Mapping[str, Any]:
return self._inputs
KiaraException (Exception)
¶
Source code in kiara/exceptions.py
class KiaraException(Exception):
pass
KiaraModuleConfigException (Exception)
¶
Source code in kiara/exceptions.py
class KiaraModuleConfigException(Exception):
def __init__(
self,
msg: str,
module_cls: Type["KiaraModule"],
config: Mapping[str, Any],
parent: Union[Exception, None] = None,
):
self._module_cls = module_cls
self._config = config
self._parent: Union[Exception, None] = parent
if not msg.endswith("."):
_msg = msg + "."
else:
_msg = msg
super().__init__(_msg)
KiaraProcessingException (Exception)
¶
Source code in kiara/exceptions.py
class KiaraProcessingException(Exception):
def __init__(
self,
msg: Union[str, Exception],
module: Union["KiaraModule", None] = None,
inputs: Union[Mapping[str, "Value"], None] = None,
):
self._module: Union["KiaraModule", None] = module
self._inputs: Union[Mapping[str, Value], None] = inputs
if isinstance(msg, Exception):
self._parent: Union[Exception, None] = msg
_msg = str(msg)
else:
self._parent = None
_msg = msg
super().__init__(_msg)
@property
def module(self) -> "KiaraModule":
return self._module # type: ignore
@property
def inputs(self) -> Mapping[str, "Value"]:
return self._inputs # type: ignore
@property
def parent_exception(self) -> Union[Exception, None]:
return self._parent
KiaraValueException (Exception)
¶
Source code in kiara/exceptions.py
class KiaraValueException(Exception):
def __init__(
self,
data_type: Type["DataType"],
value_data: Any,
exception: Exception,
):
self._data_type: Type["DataType"] = data_type
self._value_data: Any = value_data
self._exception: Exception = exception
exc_msg = str(self._exception)
if not exc_msg:
exc_msg = "no details available"
super().__init__(f"Invalid value of type '{data_type._data_type_name}': {exc_msg}") # type: ignore
NoSuchExecutionTargetException (Exception)
¶
Source code in kiara/exceptions.py
class NoSuchExecutionTargetException(Exception):
def __init__(
self,
selected_target: str,
available_targets: Iterable[str],
msg: Union[str, None] = None,
):
if msg is None:
msg = f"Specified run target '{selected_target}' is an operation, additional module configuration is not allowed."
self.avaliable_targets: Iterable[str] = available_targets
super().__init__(msg)
NoSuchValueAliasException (NoSuchValueException)
¶
Source code in kiara/exceptions.py
class NoSuchValueAliasException(NoSuchValueException):
def __init__(self, alias: str, msg: Union[str, None] = None):
self.value_id: uuid.UUID
if not msg:
msg = f"No value with alias: {alias}."
super().__init__(msg)
NoSuchValueException (Exception)
¶
Source code in kiara/exceptions.py
class NoSuchValueException(Exception):
pass
NoSuchValueIdException (NoSuchValueException)
¶
Source code in kiara/exceptions.py
class NoSuchValueIdException(NoSuchValueException):
def __init__(self, value_id: uuid.UUID, msg: Union[str, None] = None):
self.value_id: uuid.UUID
if not msg:
msg = f"No value with id: {value_id}."
super().__init__(msg)
ValueTypeConfigException (Exception)
¶
Source code in kiara/exceptions.py
class ValueTypeConfigException(Exception):
def __init__(
self,
msg: str,
type_cls: Type["DataType"],
config: Mapping[str, Any],
parent: Union[Exception, None] = None,
):
self._type_cls = type_cls
self._config = config
self._parent: Union[Exception, None] = parent
if not msg.endswith("."):
_msg = msg + "."
else:
_msg = msg
super().__init__(_msg)
interfaces
special
¶
Implementation of interfaces for Kiara.
log
¶
Functions¶
get_console()
¶
Get a global Console instance.
Returns:
| Type | Description |
|---|---|
Console |
A console instance. |
Source code in kiara/interfaces/__init__.py
def get_console() -> Console:
"""Get a global Console instance.
Returns:
Console: A console instance.
"""
global _console
if _console is None or True:
console_width = os.environ.get("CONSOLE_WIDTH", None)
width = None
if console_width:
try:
width = int(console_width)
except Exception:
pass
_console = Console(width=width)
return _console
set_console_width(width=None, prefer_env=True)
¶
Source code in kiara/interfaces/__init__.py
def set_console_width(width: Union[int, None] = None, prefer_env: bool = True):
global _console
if prefer_env or not width:
_width: Union[None, int] = None
try:
_width = int(os.environ.get("CONSOLE_WIDTH", None)) # type: ignore
except Exception:
pass
if _width:
width = _width
if width:
try:
width = int(width)
except Exception as e:
log.debug("invalid.console_width", error=str(e))
_console = Console(width=width)
if not width:
if "google.colab" in sys.modules or "jupyter_client" in sys.modules:
width = 140
if width:
con = rich.get_console()
con.width = width
Modules¶
cli
special
¶
A command-line interface for Kiara.
CLICK_CONTEXT_SETTINGS
¶Modules¶
pipeline
special
¶commands
¶Pipeline-related subcommands for the cli.
get_pipeline_config(kiara_obj, pipeline_name_or_path)
¶Source code in kiara/interfaces/cli/pipeline/commands.py
def get_pipeline_config(kiara_obj: Kiara, pipeline_name_or_path: str) -> PipelineConfig:
if os.path.isfile(pipeline_name_or_path):
pc = PipelineConfig.from_file(pipeline_name_or_path, kiara=kiara_obj)
else:
operation: Operation = kiara_obj.operation_registry.get_operation(
pipeline_name_or_path
)
pipeline_module: PipelineModule = operation.module # type: ignore
if not pipeline_module.is_pipeline():
print()
print(
f"Specified operation id exists, but is not a pipeline: {pipeline_name_or_path}."
)
sys.exit(1)
pc = pipeline_module.config
return pc
run
¶The 'run' subcommand for the cli.
python_api
special
¶
logger
¶Classes¶
KiaraAPI
¶Public API for clients
This class wraps a [Kiara][kiara.context.kiara.Kiara] instance, and allows easy a access to tasks that are typically done by a frontend. The return types of each method are json seriable in most cases.
Can be extended for special scenarios and augmented with scenario-specific methdos (Jupyter, web-frontend, ...) .
Source code in kiara/interfaces/python_api/__init__.py
class KiaraAPI(object):
"""Public API for clients
This class wraps a [Kiara][kiara.context.kiara.Kiara] instance, and allows easy a access to tasks that are
typically done by a frontend. The return types of each method are json seriable in most cases.
Can be extended for special scenarios and augmented with scenario-specific methdos (Jupyter, web-frontend, ...)
."""
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
@property
def context(self) -> "Kiara":
"""Return the kiara context.
DON"T USE THIS! This is going away in the production release.
"""
return self._kiara
# ==================================================================================================================
# methods for module and operations info
def retrieve_module_types_info(
self, filter: Union[None, str, Iterable[str]] = None
) -> ModuleTypesInfo:
"""Retrieve information for all available module types (or a filtered subset thereof).
A module type is Python class that inherits from [KiaraModule][kiara.modules.KiaraModule], and is the basic
building block for processing pipelines. Module types are not used directly by users, Operations are. Operations
are instantiated modules (meaning: the module & some (optional) configuration).
Arguments:
filter: a string (or list of string) the returned module names have to match (all filters in case of list)
Returns:
a mapping object containing module names as keys, and information about the modules as values
"""
if filter:
title = f"Filtered modules: {filter}"
module_types_names: Iterable[str] = []
for m in self._kiara.module_registry.get_module_type_names():
match = True
for f in filter:
if f.lower() not in m.lower():
match = False
break
if match:
module_types_names.append(m) # type: ignore
else:
title = "All modules"
module_types_names = self._kiara.module_registry.get_module_type_names()
module_types = {
n: self._kiara.module_registry.get_module_class(n)
for n in module_types_names
}
module_types_info = ModuleTypesInfo.create_from_type_items( # type: ignore
kiara=self.context, group_title=title, **module_types
)
return module_types_info # type: ignore
def retrieve_module_type_info(self, module_type: str) -> ModuleTypeInfo:
"""Retrieve information about a specific module type.
This can be used to retrieve information like module documentation and configuration options.
Arguments:
module_type: the registered name of the module
Returns:
an object containing all information about a module type
"""
m_cls = self._kiara.module_registry.get_module_class(module_type)
info = ModuleTypeInfo.create_from_type_class(kiara=self.context, type_cls=m_cls)
return info
def create_operation(
self, module_type: str, module_config: Union[Mapping[str, Any], None] = None
) -> Operation:
"""Create an [Operation][kiara.models.module.operation.Operation] instance for the specified module type and (optional) config.
This can be used to get information about the operation itself, it's inputs & outputs schemas, documentation etc.
Arguments:
module_type: the registered name of the module
module_config: (Optional) configuration for the module instance.
Returns:
an Operation instance (which contains all the available information about an instantiated module)
"""
if module_config is None:
module_config = {}
mc = Manifest(module_type=module_type, module_config=module_config)
module_obj = self._kiara.create_module(mc)
return module_obj.operation
@property
def operation_ids(self) -> List[str]:
"""Get a list of all available operation ids."""
return self.get_operation_ids()
def get_operation_ids(
self, *filters: str, include_internal: bool = False
) -> List[str]:
"""Get a list of all operation ids that match the specified filter.
Arguments:
filters: a list of filters (all filters must match the operation id for the operation to be included)
include_internal: also return internal operations
"""
if not filters and include_internal:
return sorted(self.context.operation_registry.operation_ids)
else:
return sorted(
self.list_operations(*filters, include_internal=include_internal).keys()
)
def get_operation(self, operation_id: str) -> Operation:
"""Return the operation instance with the specified id.
This can be used to get information about a specific operation, like inputs/outputs scheman, documentation, etc.
Arguments:
operation_id: the operation id
Returns:
operation instance data
"""
return self.context.operation_registry.get_operation(operation_id=operation_id)
def get_operation_info(self, operation_id: str) -> OperationInfo:
"""Return the full information for the specified operation id.
This is similar to the 'get_operation' method, but returns additional information. Only use this instead of
'get_operation' if you need the additional info, as it's more expensive to get.
Arguments:
operation_id: the operation id
Returns:
augmented operation instance data
"""
op = self.context.operation_registry.get_operation(operation_id=operation_id)
op_info = OperationInfo.create_from_operation(kiara=self.context, operation=op)
return op_info
def list_operations(
self, *filters: str, include_internal: bool = False
) -> Mapping[str, Operation]:
"""List all available values, optionally filter.
Arguments:
filters: the (optional) filter strings, an operation must match all of them to be included in the result
include_internal: whether to include operations that are predominantly used internally in kiara.
Returns:
a dictionary with the operation id as key, and [kiara.models.module.operation.Operation] instance data as value
"""
operations = self.context.operation_registry.operations
if filters:
temp = {}
for op_id, op in operations.items():
match = True
for f in filters:
if f.lower() not in op_id.lower():
match = False
break
if match:
temp[op_id] = op
operations = temp
if not include_internal:
temp = {}
for op_id, op in operations.items():
if not op.operation_details.is_internal_operation:
temp[op_id] = op
operations = temp
return operations
def get_operations_info(
self, *filters, include_internal: bool = False
) -> OperationGroupInfo:
"""Retrieve information about the matching operations.
This retrieves the same list of operations as [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations],
but augments each result instance with additional information that might be useful in frontends.
'OperationInfo' objects contains augmented information on top of what 'normal' [Operation][kiara.models.module.operation.Operation] objects
hold, but they can take longer to create/resolve. If you don't need any
of the augmented information, just use the [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations] method
instead.
Arguments:
filters: the (optional) filter strings, an operation must match all of them to be included in the result
include_internal: whether to include operations that are predominantly used internally in kiara.
Returns:
a wrapper object containing a dictionary of items with value_id as key, and [kiara.interfaces.python_api.models.info.OperationInfo] as value
"""
title = "Available operations"
if filters:
title = "Filtered operations"
operations = self.list_operations(*filters, include_internal=include_internal)
ops_info = OperationGroupInfo.create_from_operations(
kiara=self.context, group_title=title, **operations
)
return ops_info
# ==================================================================================================================
# methods relating to values and data
def get_value_ids(self, **matcher_params) -> List[uuid.UUID]:
"""List all available value ids for this kiara context.
This method exists mainly so frontend can retrieve a list of all value_ids that exists on the backend without
having to look up the details of each value (like [list_values][kiara.interfaces.python_api.KiaraAPI.list_values]
does). This method can also be used with a matcher, but in this case the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values]
would be preferable in most cases, because it is called under the hood, and the performance advantage of not
having to look up value details is gone.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a list of value ids
"""
if matcher_params:
values = self.list_values(**matcher_params)
return sorted(values.keys())
else:
_values = self._kiara.data_registry.retrieve_all_available_value_ids()
return sorted(_values)
def list_values(self, **matcher_params: Any) -> Dict[uuid.UUID, Value]:
"""List all available values, optionally filter.
Retrieve information about all values that are available in the current kiara context session (both stored
and non-stored).
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a dictionary with value_id as key, and [kiara.models.values.value.Value] as value
"""
if matcher_params:
matcher = ValueMatcher.create_matcher(**matcher_params)
values = self._kiara.data_registry.find_values(matcher=matcher)
else:
# TODO: make that parallel?
values = {
k: self._kiara.data_registry.get_value(k)
for k in self._kiara.data_registry.retrieve_all_available_value_ids()
}
return values
def get_value(self, value: Union[str, ValueLink, uuid.UUID]) -> Value:
"""Retrieve a value instance with the specified id or alias.
Raises an exception if no value could be found.
Arguments:
value: a value id, alias or object that has a 'value_id' attribute.
Returns:
the Value instance
"""
return self._kiara.data_registry.get_value(value=value)
def get_value_info(self, value: Union[str, uuid.UUID, ValueLink]) -> ValueInfo:
"""Retrieve an info object for a value.
'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects
hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any
of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.get_value] method
instead.
Arguments:
value: a value id, alias or object that has a 'value_id' attribute.
Returns:
the ValueInfo instance
"""
_value = self.get_value(value=value)
return ValueInfo.create_from_instance(kiara=self._kiara, instance=_value)
def get_values_info(self, **matcher_params) -> ValuesInfo:
"""Retrieve information about the matching values.
This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values],
but augments each result value instance with additional information that might be useful in frontends.
'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects
hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any
of the augmented information, just use the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] method
instead.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a wrapper object containing the items as dictionary with value_id as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value
"""
values = self.list_values(**matcher_params)
infos = ValuesInfo.create_from_instances(
kiara=self._kiara, instances={str(k): v for k, v in values.items()}
)
return infos # type: ignore
def get_alias_names(self, **matcher_params) -> List[str]:
"""List all available alias keys.
This method exists mainly so frontend can retrieve a list of all value_ids that exists on the backend without
having to look up the details of each value (like [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases]
does). This method can also be used with a matcher, but in this case the [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases]
would be preferrable in most cases, because it is called under the hood, and the performance advantage of not
having to look up value details is gone.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a list of value ids
"""
if matcher_params:
values = self.list_aliases(**matcher_params)
return list(values.keys())
else:
_values = self._kiara.alias_registry.all_aliases
return list(_values)
def list_aliases(self, **matcher_params) -> Dict[str, Value]:
"""List all available values that have an alias assigned, optionally filter.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a dictionary with value_id as key, and [kiara.models.values.value.Value] as value
"""
if matcher_params:
matcher_params["has_alias"] = True
all_values = self.list_values(**matcher_params)
result: Dict[str, Value] = {}
for value in all_values.values():
aliases = self._kiara.alias_registry.find_aliases_for_value_id(
value_id=value.value_id
)
for a in aliases:
if a in result.keys():
raise Exception(
f"Duplicate value alias '{a}': this is most likely a bug."
)
result[a] = value
result = {k: result[k] for k in sorted(result.keys())}
else:
all_aliases = self._kiara.alias_registry.all_aliases
result = {
k: self._kiara.data_registry.get_value(f"alias:{k}")
for k in all_aliases
}
return result
def get_aliases_info(self, **matcher_params) -> ValuesInfo:
"""Retrieve information about the matching values.
This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values],
but augments each result value instance with additional information that might be useful in frontends.
'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects
hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any
of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.list_aliases] method
instead.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a dictionary with a value alias as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value
"""
values = self.list_aliases(**matcher_params)
infos = ValuesInfo.create_from_instances(
kiara=self._kiara, instances={str(k): v for k, v in values.items()}
)
return infos # type: ignore
def store_value(
self,
value: Union[str, uuid.UUID, ValueLink],
aliases: Union[str, Iterable[str], None],
):
"""Store the specified value in the (default) value store.
Arguments:
value: the value (or a reference to it)
aliases: (Optional) aliases for the value
"""
if isinstance(aliases, str):
aliases = [aliases]
value_obj = self.get_value(value)
persisted_data: Union[None, PersistedData] = None
try:
persisted_data = self.context.data_registry.store_value(value=value_obj)
if aliases:
self.context.alias_registry.register_aliases(
value_obj.value_id, *aliases
)
result = StoreValueResult.construct(
value=value_obj,
aliases=sorted(aliases) if aliases else [],
error=None,
persisted_data=persisted_data,
)
except Exception as e:
log_exception(e)
result = StoreValueResult.construct(
value=value_obj,
aliases=sorted(aliases) if aliases else [],
error=str(e),
persisted_data=persisted_data,
)
return result
def store_values(
self,
values: Mapping[str, Union[str, uuid.UUID, ValueLink]],
alias_map: Mapping[str, Iterable[str]],
) -> StoreValuesResult:
"""Store multiple values into the (default) kiara value store.
Values are identified by unique keys in both input arguments, the alias map references the key that is used in
the 'values' argument.
Arguments:
values: a map of value keys/values
alias_map: a map of value keys aliases
Returns:
an object outlining which values (identified by the specified value key) where stored and how
"""
result = {}
for field_name, value in values.items():
aliases = alias_map.get(field_name)
value_obj = self.get_value(value)
store_result = self.store_value(value=value_obj, aliases=aliases)
result[field_name] = store_result
return StoreValuesResult.construct(__root__=result)
# ------------------------------------------------------------------------------------------------------------------
# operation-related methods
def get_operation_type(self, op_type: Union[str, Type[OP_TYPE]]):
"""Get the management object for the specified operation type."""
return self.context.operation_registry.get_operation_type(op_type=op_type)
# ------------------------------------------------------------------------------------------------------------------
# pipeline-related methods
def assemble_filter_pipeline_config(
self,
data_type: str,
filters: Union[str, Iterable[str], Mapping[str, str]],
endpoint: Union[None, Manifest, str] = None,
endpoint_input_field: Union[str, None] = None,
endpoint_step_id: Union[str, None] = None,
extra_input_aliases: Union[None, Mapping[str, str]] = None,
extra_output_aliases: Union[None, Mapping[str, str]] = None,
) -> PipelineConfig:
"""Assemble a (pipeline) module config to filter values of a specific data type.
Optionally, a module that uses the filtered dataset as input can be specified.
# TODO: document filter names
For the 'filters' argument, the accepted inputs are:
- a string, in which case a single-step pipeline will be created, with the string referencing the operation id or filter
- a list of strings: in which case a multi-step pipeline will be created, the step_ids will be calculated automatically
- a map of string pairs: the keys are step ids, the values operation ids or filter names
Arguments:
data_type: the type of the data to filter
filters: a list of operation ids or filter names (and potentiall step_ids if type is a mapping)
endpoint: optional module to put as last step in the created pipeline
endpoing_input_field: field name of the input that will receive the filtered value
endpoint_step_id: id to use for the endpoint step (module type name will be used if not provided)
extra_input_aliases: extra output aliases to add to the pipeline config
extra_output_aliases: extra output aliases to add to the pipeline config
Returns:
the (pipeline) module configuration of the filter pipeline
"""
filter_op_type: FilterOperationType = self._kiara.operation_registry.get_operation_type("filter") # type: ignore
pipeline_config = filter_op_type.assemble_filter_pipeline_config(
data_type=data_type,
filters=filters,
endpoint=endpoint,
endpoint_input_field=endpoint_input_field,
endpoint_step_id=endpoint_step_id,
extra_input_aliases=extra_input_aliases,
extra_output_aliases=extra_output_aliases,
)
return pipeline_config
def assemble_render_pipeline(
self,
data_type: str,
target_format: Union[str, Iterable[str]] = "string",
filters: Union[None, str, Iterable[str], Mapping[str, str]] = None,
) -> Operation:
"""Create a manifest describing a transformation that renders a value of the specified data type in the target format.
If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches
the value type of the source value, and the provided target format.
Arguments:
value: the value (or value id)
target_format: the format into which to render the value
Returns:
the manifest for the transformation
"""
render_op_type: RenderValueOperationType = self._kiara.operation_registry.get_operation_type( # type: ignore
"render_value"
) # type: ignore
ops = render_op_type.get_render_operations_for_source_type(data_type)
if isinstance(target_format, str):
target_format = [target_format]
match = None
for _target_type in target_format:
if _target_type not in ops.keys():
continue
match = ops[_target_type]
break
if not match:
if not ops:
msg = f"No render operations registered for source type '{data_type}'."
else:
msg = f"Registered target types for source type '{data_type}': {', '.join(ops.keys())}."
raise Exception(
f"No render operation for source type '{data_type}' to target type(s) registered: '{', '.join(target_format)}'. {msg}"
)
if filters:
# filter_op_type: FilterOperationType = self._kiara.operation_registry.get_operation_type("filter") # type: ignore
endpoint = Manifest(
module_type=match.module_type, module_config=match.module_config
)
extra_input_aliases = {"render_value.render_config": "render_config"}
extra_output_aliases = {
"render_value.render_value_result": "render_value_result"
}
pipeline_config = self.assemble_filter_pipeline_config(
data_type=data_type,
filters=filters,
endpoint=endpoint,
endpoint_input_field="value",
endpoint_step_id="render_value",
extra_input_aliases=extra_input_aliases,
extra_output_aliases=extra_output_aliases,
)
manifest = Manifest(
module_type="pipeline", module_config=pipeline_config.dict()
)
module = self._kiara.module_registry.create_module(manifest=manifest)
operation = Operation.create_from_module(module, doc=pipeline_config.doc)
else:
operation = match
return operation
# ------------------------------------------------------------------------------------------------------------------
# job-related methods
def queue_manifest(
self, manifest: Manifest, inputs: Union[None, Mapping[str, Any]] = None
) -> uuid.UUID:
"""Queue a job using the provided manifest to describe the module and config that should be executed.
Arguments:
manifest: the manifest
inputs: the job inputs (can be either references to values, or raw inputs
Returns:
a result value map instance
"""
if inputs is None:
inputs = {}
job_config = self.context.job_registry.prepare_job_config(
manifest=manifest, inputs=inputs
)
job_id = self.context.job_registry.execute_job(
job_config=job_config, wait=False
)
return job_id
def run_manifest(
self, manifest: Manifest, inputs: Union[None, Mapping[str, Any]] = None
) -> ValueMap:
"""Run a job using the provided manifest to describe the module and config that should be executed.
Arguments:
manifest: the manifest
inputs: the job inputs (can be either references to values, or raw inputs
Returns:
a result value map instance
"""
job_id = self.queue_manifest(manifest=manifest, inputs=inputs)
return self.context.job_registry.retrieve_result(job_id=job_id)
def queue_job(
self,
operation: Union[str, Path, Manifest],
inputs: Mapping[str, Any],
operation_config: Union[None, Mapping[str, Any]] = None,
) -> uuid.UUID:
"""Queue a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result.
This is a convenience method that auto-detects what is meant by the 'operation' string input argument.
Arguments:
operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found)..
inputs: the operation inputs
operation_config: the (optional) module config in case 'operation' is a module name
Returns:
the queued job id
"""
if isinstance(operation, Path):
if not operation.is_file():
raise Exception(
f"Can't queue job from file '{operation.as_posix()}': file does not exist."
)
operation = operation.as_posix()
if not isinstance(operation, Manifest):
manifest: Manifest = create_operation(
module_or_operation=operation,
operation_config=operation_config,
kiara=self.context,
)
else:
manifest = operation
job_id = self.queue_manifest(manifest=manifest, inputs=inputs)
return job_id
def run_job(
self,
operation: Union[str, Path, Manifest],
inputs: Mapping[str, Any],
operation_config: Union[None, Mapping[str, Any]] = None,
) -> ValueMap:
"""Run a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result.
This is a convenience method that auto-detects what is meant by the 'operation' string input argument.
In general, try to avoid this method and use 'queue_job', 'get_job' and 'retrieve_job_result' manually instead,
since this is a blocking operation.
Arguments:
operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found)..
inputs: the operation inputs
operation_config: the (optional) module config in case 'operation' is a module name
Returns:
the job result value map
"""
job_id = self.queue_job(
operation=operation, inputs=inputs, operation_config=operation_config
)
return self.context.job_registry.retrieve_result(job_id=job_id)
def get_job(self, job_id: Union[str, uuid.UUID]) -> ActiveJob:
"""Retrieve the status of the job with the provided id."""
if isinstance(job_id, str):
job_id = uuid.UUID(job_id)
job_status = self.context.job_registry.get_job(job_id=job_id)
return job_status
def retrieve_job_result(self, job_id: Union[str, uuid.UUID]) -> ValueMap:
"""Retrieve the result(s) of the specified job."""
if isinstance(job_id, str):
job_id = uuid.UUID(job_id)
result = self.context.job_registry.retrieve_result(job_id=job_id)
return result
def render_value(
self,
value: Union[str, uuid.UUID, ValueLink],
target_format: Union[str, Iterable[str]] = "string",
filters: Union[None, Iterable[str], Mapping[str, str]] = None,
render_config: Union[Mapping[str, str], None] = None,
) -> RenderValueResult:
"""Render a value in the specified target format.
If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches
the value type of the source value, and the provided target format.
Arguments:
value: the value (or value id)
target_format: the format into which to render the value
filters: an (optional) list of filters
render_config: manifest specific render configuration
Returns:
the rendered value data
"""
_value = self.get_value(value)
render_operation = self.assemble_render_pipeline(
data_type=_value.data_type_name,
target_format=target_format,
filters=filters,
)
if render_config and "render_config" in render_config.keys():
# raise NotImplementedError()
# TODO: is this necessary?
render_config = render_config["render_config"] # type: ignore
# manifest_hash = render_config["manifest_hash"]
# if manifest_hash != render_operation.manifest_hash:
# raise NotImplementedError(
# "Using a non-default render operation is not supported (yet)."
# )
# render_config = render_config["render_config"]
result = render_operation.run(
kiara=self.context,
inputs={"value": _value, "render_config": render_config},
)
render_result = result["render_value_result"]
if render_result.data_type_name != "render_value_result":
raise Exception(
f"Invalid result type for render operation: {render_result.data_type_name}"
)
return render_result.data
context: Kiara
property
readonly
¶Return the kiara context.
DON"T USE THIS! This is going away in the production release.
operation_ids: List[str]
property
readonly
¶Get a list of all available operation ids.
assemble_filter_pipeline_config(self, data_type, filters, endpoint=None, endpoint_input_field=None, endpoint_step_id=None, extra_input_aliases=None, extra_output_aliases=None)
¶Assemble a (pipeline) module config to filter values of a specific data type.
Optionally, a module that uses the filtered dataset as input can be specified.
TODO: document filter names¶
For the 'filters' argument, the accepted inputs are: - a string, in which case a single-step pipeline will be created, with the string referencing the operation id or filter - a list of strings: in which case a multi-step pipeline will be created, the step_ids will be calculated automatically - a map of string pairs: the keys are step ids, the values operation ids or filter names
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data_type |
str |
the type of the data to filter |
required |
filters |
Union[str, Iterable[str], Mapping[str, str]] |
a list of operation ids or filter names (and potentiall step_ids if type is a mapping) |
required |
endpoint |
Union[NoneType, kiara.models.module.manifest.Manifest, str] |
optional module to put as last step in the created pipeline |
None |
endpoing_input_field |
field name of the input that will receive the filtered value |
required | |
endpoint_step_id |
Optional[str] |
id to use for the endpoint step (module type name will be used if not provided) |
None |
extra_input_aliases |
Optional[Mapping[str, str]] |
extra output aliases to add to the pipeline config |
None |
extra_output_aliases |
Optional[Mapping[str, str]] |
extra output aliases to add to the pipeline config |
None |
Returns:
| Type | Description |
|---|---|
PipelineConfig |
the (pipeline) module configuration of the filter pipeline |
Source code in kiara/interfaces/python_api/__init__.py
def assemble_filter_pipeline_config(
self,
data_type: str,
filters: Union[str, Iterable[str], Mapping[str, str]],
endpoint: Union[None, Manifest, str] = None,
endpoint_input_field: Union[str, None] = None,
endpoint_step_id: Union[str, None] = None,
extra_input_aliases: Union[None, Mapping[str, str]] = None,
extra_output_aliases: Union[None, Mapping[str, str]] = None,
) -> PipelineConfig:
"""Assemble a (pipeline) module config to filter values of a specific data type.
Optionally, a module that uses the filtered dataset as input can be specified.
# TODO: document filter names
For the 'filters' argument, the accepted inputs are:
- a string, in which case a single-step pipeline will be created, with the string referencing the operation id or filter
- a list of strings: in which case a multi-step pipeline will be created, the step_ids will be calculated automatically
- a map of string pairs: the keys are step ids, the values operation ids or filter names
Arguments:
data_type: the type of the data to filter
filters: a list of operation ids or filter names (and potentiall step_ids if type is a mapping)
endpoint: optional module to put as last step in the created pipeline
endpoing_input_field: field name of the input that will receive the filtered value
endpoint_step_id: id to use for the endpoint step (module type name will be used if not provided)
extra_input_aliases: extra output aliases to add to the pipeline config
extra_output_aliases: extra output aliases to add to the pipeline config
Returns:
the (pipeline) module configuration of the filter pipeline
"""
filter_op_type: FilterOperationType = self._kiara.operation_registry.get_operation_type("filter") # type: ignore
pipeline_config = filter_op_type.assemble_filter_pipeline_config(
data_type=data_type,
filters=filters,
endpoint=endpoint,
endpoint_input_field=endpoint_input_field,
endpoint_step_id=endpoint_step_id,
extra_input_aliases=extra_input_aliases,
extra_output_aliases=extra_output_aliases,
)
return pipeline_config
assemble_render_pipeline(self, data_type, target_format='string', filters=None)
¶Create a manifest describing a transformation that renders a value of the specified data type in the target format.
If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches the value type of the source value, and the provided target format.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
the value (or value id) |
required | |
target_format |
Union[str, Iterable[str]] |
the format into which to render the value |
'string' |
Returns:
| Type | Description |
|---|---|
Operation |
the manifest for the transformation |
Source code in kiara/interfaces/python_api/__init__.py
def assemble_render_pipeline(
self,
data_type: str,
target_format: Union[str, Iterable[str]] = "string",
filters: Union[None, str, Iterable[str], Mapping[str, str]] = None,
) -> Operation:
"""Create a manifest describing a transformation that renders a value of the specified data type in the target format.
If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches
the value type of the source value, and the provided target format.
Arguments:
value: the value (or value id)
target_format: the format into which to render the value
Returns:
the manifest for the transformation
"""
render_op_type: RenderValueOperationType = self._kiara.operation_registry.get_operation_type( # type: ignore
"render_value"
) # type: ignore
ops = render_op_type.get_render_operations_for_source_type(data_type)
if isinstance(target_format, str):
target_format = [target_format]
match = None
for _target_type in target_format:
if _target_type not in ops.keys():
continue
match = ops[_target_type]
break
if not match:
if not ops:
msg = f"No render operations registered for source type '{data_type}'."
else:
msg = f"Registered target types for source type '{data_type}': {', '.join(ops.keys())}."
raise Exception(
f"No render operation for source type '{data_type}' to target type(s) registered: '{', '.join(target_format)}'. {msg}"
)
if filters:
# filter_op_type: FilterOperationType = self._kiara.operation_registry.get_operation_type("filter") # type: ignore
endpoint = Manifest(
module_type=match.module_type, module_config=match.module_config
)
extra_input_aliases = {"render_value.render_config": "render_config"}
extra_output_aliases = {
"render_value.render_value_result": "render_value_result"
}
pipeline_config = self.assemble_filter_pipeline_config(
data_type=data_type,
filters=filters,
endpoint=endpoint,
endpoint_input_field="value",
endpoint_step_id="render_value",
extra_input_aliases=extra_input_aliases,
extra_output_aliases=extra_output_aliases,
)
manifest = Manifest(
module_type="pipeline", module_config=pipeline_config.dict()
)
module = self._kiara.module_registry.create_module(manifest=manifest)
operation = Operation.create_from_module(module, doc=pipeline_config.doc)
else:
operation = match
return operation
create_operation(self, module_type, module_config=None)
¶Create an Operation instance for the specified module type and (optional) config.
This can be used to get information about the operation itself, it's inputs & outputs schemas, documentation etc.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
module_type |
str |
the registered name of the module |
required |
module_config |
Optional[Mapping[str, Any]] |
(Optional) configuration for the module instance. |
None |
Returns:
| Type | Description |
|---|---|
Operation |
an Operation instance (which contains all the available information about an instantiated module) |
Source code in kiara/interfaces/python_api/__init__.py
def create_operation(
self, module_type: str, module_config: Union[Mapping[str, Any], None] = None
) -> Operation:
"""Create an [Operation][kiara.models.module.operation.Operation] instance for the specified module type and (optional) config.
This can be used to get information about the operation itself, it's inputs & outputs schemas, documentation etc.
Arguments:
module_type: the registered name of the module
module_config: (Optional) configuration for the module instance.
Returns:
an Operation instance (which contains all the available information about an instantiated module)
"""
if module_config is None:
module_config = {}
mc = Manifest(module_type=module_type, module_config=module_config)
module_obj = self._kiara.create_module(mc)
return module_obj.operation
get_alias_names(self, **matcher_params)
¶List all available alias keys.
This method exists mainly so frontend can retrieve a list of all value_ids that exists on the backend without having to look up the details of each value (like list_aliases does). This method can also be used with a matcher, but in this case the list_aliases would be preferrable in most cases, because it is called under the hood, and the performance advantage of not having to look up value details is gone.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
matcher_params |
the (optional) filter parameters, check the ValueMatcher class for available parameters |
{} |
Returns:
| Type | Description |
|---|---|
List[str] |
a list of value ids |
Source code in kiara/interfaces/python_api/__init__.py
def get_alias_names(self, **matcher_params) -> List[str]:
"""List all available alias keys.
This method exists mainly so frontend can retrieve a list of all value_ids that exists on the backend without
having to look up the details of each value (like [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases]
does). This method can also be used with a matcher, but in this case the [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases]
would be preferrable in most cases, because it is called under the hood, and the performance advantage of not
having to look up value details is gone.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a list of value ids
"""
if matcher_params:
values = self.list_aliases(**matcher_params)
return list(values.keys())
else:
_values = self._kiara.alias_registry.all_aliases
return list(_values)
get_aliases_info(self, **matcher_params)
¶Retrieve information about the matching values.
This retrieves the same list of values as list_values, but augments each result value instance with additional information that might be useful in frontends.
'ValueInfo' objects contains augmented information on top of what 'normal' Value objects hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any of the augmented information, just use the get_value method instead.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
matcher_params |
the (optional) filter parameters, check the ValueMatcher class for available parameters |
{} |
Returns:
| Type | Description |
|---|---|
ValuesInfo |
a dictionary with a value alias as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value |
Source code in kiara/interfaces/python_api/__init__.py
def get_aliases_info(self, **matcher_params) -> ValuesInfo:
"""Retrieve information about the matching values.
This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values],
but augments each result value instance with additional information that might be useful in frontends.
'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects
hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any
of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.list_aliases] method
instead.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a dictionary with a value alias as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value
"""
values = self.list_aliases(**matcher_params)
infos = ValuesInfo.create_from_instances(
kiara=self._kiara, instances={str(k): v for k, v in values.items()}
)
return infos # type: ignore
get_job(self, job_id)
¶Retrieve the status of the job with the provided id.
Source code in kiara/interfaces/python_api/__init__.py
def get_job(self, job_id: Union[str, uuid.UUID]) -> ActiveJob:
"""Retrieve the status of the job with the provided id."""
if isinstance(job_id, str):
job_id = uuid.UUID(job_id)
job_status = self.context.job_registry.get_job(job_id=job_id)
return job_status
get_operation(self, operation_id)
¶Return the operation instance with the specified id.
This can be used to get information about a specific operation, like inputs/outputs scheman, documentation, etc.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
operation_id |
str |
the operation id |
required |
Returns:
| Type | Description |
|---|---|
Operation |
operation instance data |
Source code in kiara/interfaces/python_api/__init__.py
def get_operation(self, operation_id: str) -> Operation:
"""Return the operation instance with the specified id.
This can be used to get information about a specific operation, like inputs/outputs scheman, documentation, etc.
Arguments:
operation_id: the operation id
Returns:
operation instance data
"""
return self.context.operation_registry.get_operation(operation_id=operation_id)
get_operation_ids(self, *filters, *, include_internal=False)
¶Get a list of all operation ids that match the specified filter.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filters |
str |
a list of filters (all filters must match the operation id for the operation to be included) |
() |
include_internal |
bool |
also return internal operations |
False |
Source code in kiara/interfaces/python_api/__init__.py
def get_operation_ids(
self, *filters: str, include_internal: bool = False
) -> List[str]:
"""Get a list of all operation ids that match the specified filter.
Arguments:
filters: a list of filters (all filters must match the operation id for the operation to be included)
include_internal: also return internal operations
"""
if not filters and include_internal:
return sorted(self.context.operation_registry.operation_ids)
else:
return sorted(
self.list_operations(*filters, include_internal=include_internal).keys()
)
get_operation_info(self, operation_id)
¶Return the full information for the specified operation id.
This is similar to the 'get_operation' method, but returns additional information. Only use this instead of 'get_operation' if you need the additional info, as it's more expensive to get.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
operation_id |
str |
the operation id |
required |
Returns:
| Type | Description |
|---|---|
OperationInfo |
augmented operation instance data |
Source code in kiara/interfaces/python_api/__init__.py
def get_operation_info(self, operation_id: str) -> OperationInfo:
"""Return the full information for the specified operation id.
This is similar to the 'get_operation' method, but returns additional information. Only use this instead of
'get_operation' if you need the additional info, as it's more expensive to get.
Arguments:
operation_id: the operation id
Returns:
augmented operation instance data
"""
op = self.context.operation_registry.get_operation(operation_id=operation_id)
op_info = OperationInfo.create_from_operation(kiara=self.context, operation=op)
return op_info
get_operation_type(self, op_type)
¶Get the management object for the specified operation type.
Source code in kiara/interfaces/python_api/__init__.py
def get_operation_type(self, op_type: Union[str, Type[OP_TYPE]]):
"""Get the management object for the specified operation type."""
return self.context.operation_registry.get_operation_type(op_type=op_type)
get_operations_info(self, *filters, *, include_internal=False)
¶Retrieve information about the matching operations.
This retrieves the same list of operations as list_operations, but augments each result instance with additional information that might be useful in frontends.
'OperationInfo' objects contains augmented information on top of what 'normal' Operation objects hold, but they can take longer to create/resolve. If you don't need any of the augmented information, just use the list_operations method instead.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filters |
the (optional) filter strings, an operation must match all of them to be included in the result |
() |
|
include_internal |
bool |
whether to include operations that are predominantly used internally in kiara. |
False |
Returns:
| Type | Description |
|---|---|
OperationGroupInfo |
a wrapper object containing a dictionary of items with value_id as key, and [kiara.interfaces.python_api.models.info.OperationInfo] as value |
Source code in kiara/interfaces/python_api/__init__.py
def get_operations_info(
self, *filters, include_internal: bool = False
) -> OperationGroupInfo:
"""Retrieve information about the matching operations.
This retrieves the same list of operations as [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations],
but augments each result instance with additional information that might be useful in frontends.
'OperationInfo' objects contains augmented information on top of what 'normal' [Operation][kiara.models.module.operation.Operation] objects
hold, but they can take longer to create/resolve. If you don't need any
of the augmented information, just use the [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations] method
instead.
Arguments:
filters: the (optional) filter strings, an operation must match all of them to be included in the result
include_internal: whether to include operations that are predominantly used internally in kiara.
Returns:
a wrapper object containing a dictionary of items with value_id as key, and [kiara.interfaces.python_api.models.info.OperationInfo] as value
"""
title = "Available operations"
if filters:
title = "Filtered operations"
operations = self.list_operations(*filters, include_internal=include_internal)
ops_info = OperationGroupInfo.create_from_operations(
kiara=self.context, group_title=title, **operations
)
return ops_info
get_value(self, value)
¶Retrieve a value instance with the specified id or alias.
Raises an exception if no value could be found.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Union[str, kiara.registries.data.ValueLink, uuid.UUID] |
a value id, alias or object that has a 'value_id' attribute. |
required |
Returns:
| Type | Description |
|---|---|
Value |
the Value instance |
Source code in kiara/interfaces/python_api/__init__.py
def get_value(self, value: Union[str, ValueLink, uuid.UUID]) -> Value:
"""Retrieve a value instance with the specified id or alias.
Raises an exception if no value could be found.
Arguments:
value: a value id, alias or object that has a 'value_id' attribute.
Returns:
the Value instance
"""
return self._kiara.data_registry.get_value(value=value)
get_value_ids(self, **matcher_params)
¶List all available value ids for this kiara context.
This method exists mainly so frontend can retrieve a list of all value_ids that exists on the backend without having to look up the details of each value (like list_values does). This method can also be used with a matcher, but in this case the list_values would be preferable in most cases, because it is called under the hood, and the performance advantage of not having to look up value details is gone.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
matcher_params |
the (optional) filter parameters, check the ValueMatcher class for available parameters |
{} |
Returns:
| Type | Description |
|---|---|
List[uuid.UUID] |
a list of value ids |
Source code in kiara/interfaces/python_api/__init__.py
def get_value_ids(self, **matcher_params) -> List[uuid.UUID]:
"""List all available value ids for this kiara context.
This method exists mainly so frontend can retrieve a list of all value_ids that exists on the backend without
having to look up the details of each value (like [list_values][kiara.interfaces.python_api.KiaraAPI.list_values]
does). This method can also be used with a matcher, but in this case the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values]
would be preferable in most cases, because it is called under the hood, and the performance advantage of not
having to look up value details is gone.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a list of value ids
"""
if matcher_params:
values = self.list_values(**matcher_params)
return sorted(values.keys())
else:
_values = self._kiara.data_registry.retrieve_all_available_value_ids()
return sorted(_values)
get_value_info(self, value)
¶Retrieve an info object for a value.
'ValueInfo' objects contains augmented information on top of what 'normal' Value objects hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any of the augmented information, just use the get_value method instead.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Union[str, uuid.UUID, kiara.registries.data.ValueLink] |
a value id, alias or object that has a 'value_id' attribute. |
required |
Returns:
| Type | Description |
|---|---|
ValueInfo |
the ValueInfo instance |
Source code in kiara/interfaces/python_api/__init__.py
def get_value_info(self, value: Union[str, uuid.UUID, ValueLink]) -> ValueInfo:
"""Retrieve an info object for a value.
'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects
hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any
of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.get_value] method
instead.
Arguments:
value: a value id, alias or object that has a 'value_id' attribute.
Returns:
the ValueInfo instance
"""
_value = self.get_value(value=value)
return ValueInfo.create_from_instance(kiara=self._kiara, instance=_value)
get_values_info(self, **matcher_params)
¶Retrieve information about the matching values.
This retrieves the same list of values as list_values, but augments each result value instance with additional information that might be useful in frontends.
'ValueInfo' objects contains augmented information on top of what 'normal' Value objects hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any of the augmented information, just use the list_values method instead.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
matcher_params |
the (optional) filter parameters, check the ValueMatcher class for available parameters |
{} |
Returns:
| Type | Description |
|---|---|
ValuesInfo |
a wrapper object containing the items as dictionary with value_id as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value |
Source code in kiara/interfaces/python_api/__init__.py
def get_values_info(self, **matcher_params) -> ValuesInfo:
"""Retrieve information about the matching values.
This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values],
but augments each result value instance with additional information that might be useful in frontends.
'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects
hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any
of the augmented information, just use the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] method
instead.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a wrapper object containing the items as dictionary with value_id as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value
"""
values = self.list_values(**matcher_params)
infos = ValuesInfo.create_from_instances(
kiara=self._kiara, instances={str(k): v for k, v in values.items()}
)
return infos # type: ignore
list_aliases(self, **matcher_params)
¶List all available values that have an alias assigned, optionally filter.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
matcher_params |
the (optional) filter parameters, check the ValueMatcher class for available parameters |
{} |
Returns:
| Type | Description |
|---|---|
Dict[str, kiara.models.values.value.Value] |
a dictionary with value_id as key, and [kiara.models.values.value.Value] as value |
Source code in kiara/interfaces/python_api/__init__.py
def list_aliases(self, **matcher_params) -> Dict[str, Value]:
"""List all available values that have an alias assigned, optionally filter.
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a dictionary with value_id as key, and [kiara.models.values.value.Value] as value
"""
if matcher_params:
matcher_params["has_alias"] = True
all_values = self.list_values(**matcher_params)
result: Dict[str, Value] = {}
for value in all_values.values():
aliases = self._kiara.alias_registry.find_aliases_for_value_id(
value_id=value.value_id
)
for a in aliases:
if a in result.keys():
raise Exception(
f"Duplicate value alias '{a}': this is most likely a bug."
)
result[a] = value
result = {k: result[k] for k in sorted(result.keys())}
else:
all_aliases = self._kiara.alias_registry.all_aliases
result = {
k: self._kiara.data_registry.get_value(f"alias:{k}")
for k in all_aliases
}
return result
list_operations(self, *filters, *, include_internal=False)
¶List all available values, optionally filter.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filters |
str |
the (optional) filter strings, an operation must match all of them to be included in the result |
() |
include_internal |
bool |
whether to include operations that are predominantly used internally in kiara. |
False |
Returns:
| Type | Description |
|---|---|
Mapping[str, kiara.models.module.operation.Operation] |
a dictionary with the operation id as key, and [kiara.models.module.operation.Operation] instance data as value |
Source code in kiara/interfaces/python_api/__init__.py
def list_operations(
self, *filters: str, include_internal: bool = False
) -> Mapping[str, Operation]:
"""List all available values, optionally filter.
Arguments:
filters: the (optional) filter strings, an operation must match all of them to be included in the result
include_internal: whether to include operations that are predominantly used internally in kiara.
Returns:
a dictionary with the operation id as key, and [kiara.models.module.operation.Operation] instance data as value
"""
operations = self.context.operation_registry.operations
if filters:
temp = {}
for op_id, op in operations.items():
match = True
for f in filters:
if f.lower() not in op_id.lower():
match = False
break
if match:
temp[op_id] = op
operations = temp
if not include_internal:
temp = {}
for op_id, op in operations.items():
if not op.operation_details.is_internal_operation:
temp[op_id] = op
operations = temp
return operations
list_values(self, **matcher_params)
¶List all available values, optionally filter.
Retrieve information about all values that are available in the current kiara context session (both stored and non-stored).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
matcher_params |
Any |
the (optional) filter parameters, check the ValueMatcher class for available parameters |
{} |
Returns:
| Type | Description |
|---|---|
Dict[uuid.UUID, kiara.models.values.value.Value] |
a dictionary with value_id as key, and [kiara.models.values.value.Value] as value |
Source code in kiara/interfaces/python_api/__init__.py
def list_values(self, **matcher_params: Any) -> Dict[uuid.UUID, Value]:
"""List all available values, optionally filter.
Retrieve information about all values that are available in the current kiara context session (both stored
and non-stored).
Arguments:
matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters
Returns:
a dictionary with value_id as key, and [kiara.models.values.value.Value] as value
"""
if matcher_params:
matcher = ValueMatcher.create_matcher(**matcher_params)
values = self._kiara.data_registry.find_values(matcher=matcher)
else:
# TODO: make that parallel?
values = {
k: self._kiara.data_registry.get_value(k)
for k in self._kiara.data_registry.retrieve_all_available_value_ids()
}
return values
queue_job(self, operation, inputs, operation_config=None)
¶Queue a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result.
This is a convenience method that auto-detects what is meant by the 'operation' string input argument.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
operation |
Union[str, pathlib.Path, kiara.models.module.manifest.Manifest] |
a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found).. |
required |
inputs |
Mapping[str, Any] |
the operation inputs |
required |
operation_config |
Optional[Mapping[str, Any]] |
the (optional) module config in case 'operation' is a module name |
None |
Returns:
| Type | Description |
|---|---|
UUID |
the queued job id |
Source code in kiara/interfaces/python_api/__init__.py
def queue_job(
self,
operation: Union[str, Path, Manifest],
inputs: Mapping[str, Any],
operation_config: Union[None, Mapping[str, Any]] = None,
) -> uuid.UUID:
"""Queue a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result.
This is a convenience method that auto-detects what is meant by the 'operation' string input argument.
Arguments:
operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found)..
inputs: the operation inputs
operation_config: the (optional) module config in case 'operation' is a module name
Returns:
the queued job id
"""
if isinstance(operation, Path):
if not operation.is_file():
raise Exception(
f"Can't queue job from file '{operation.as_posix()}': file does not exist."
)
operation = operation.as_posix()
if not isinstance(operation, Manifest):
manifest: Manifest = create_operation(
module_or_operation=operation,
operation_config=operation_config,
kiara=self.context,
)
else:
manifest = operation
job_id = self.queue_manifest(manifest=manifest, inputs=inputs)
return job_id
queue_manifest(self, manifest, inputs=None)
¶Queue a job using the provided manifest to describe the module and config that should be executed.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
manifest |
Manifest |
the manifest |
required |
inputs |
Optional[Mapping[str, Any]] |
the job inputs (can be either references to values, or raw inputs |
None |
Returns:
| Type | Description |
|---|---|
UUID |
a result value map instance |
Source code in kiara/interfaces/python_api/__init__.py
def queue_manifest(
self, manifest: Manifest, inputs: Union[None, Mapping[str, Any]] = None
) -> uuid.UUID:
"""Queue a job using the provided manifest to describe the module and config that should be executed.
Arguments:
manifest: the manifest
inputs: the job inputs (can be either references to values, or raw inputs
Returns:
a result value map instance
"""
if inputs is None:
inputs = {}
job_config = self.context.job_registry.prepare_job_config(
manifest=manifest, inputs=inputs
)
job_id = self.context.job_registry.execute_job(
job_config=job_config, wait=False
)
return job_id
render_value(self, value, target_format='string', filters=None, render_config=None)
¶Render a value in the specified target format.
If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches the value type of the source value, and the provided target format.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Union[str, uuid.UUID, kiara.registries.data.ValueLink] |
the value (or value id) |
required |
target_format |
Union[str, Iterable[str]] |
the format into which to render the value |
'string' |
filters |
Union[NoneType, Iterable[str], Mapping[str, str]] |
an (optional) list of filters |
None |
render_config |
Optional[Mapping[str, str]] |
manifest specific render configuration |
None |
Returns:
| Type | Description |
|---|---|
RenderValueResult |
the rendered value data |
Source code in kiara/interfaces/python_api/__init__.py
def render_value(
self,
value: Union[str, uuid.UUID, ValueLink],
target_format: Union[str, Iterable[str]] = "string",
filters: Union[None, Iterable[str], Mapping[str, str]] = None,
render_config: Union[Mapping[str, str], None] = None,
) -> RenderValueResult:
"""Render a value in the specified target format.
If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches
the value type of the source value, and the provided target format.
Arguments:
value: the value (or value id)
target_format: the format into which to render the value
filters: an (optional) list of filters
render_config: manifest specific render configuration
Returns:
the rendered value data
"""
_value = self.get_value(value)
render_operation = self.assemble_render_pipeline(
data_type=_value.data_type_name,
target_format=target_format,
filters=filters,
)
if render_config and "render_config" in render_config.keys():
# raise NotImplementedError()
# TODO: is this necessary?
render_config = render_config["render_config"] # type: ignore
# manifest_hash = render_config["manifest_hash"]
# if manifest_hash != render_operation.manifest_hash:
# raise NotImplementedError(
# "Using a non-default render operation is not supported (yet)."
# )
# render_config = render_config["render_config"]
result = render_operation.run(
kiara=self.context,
inputs={"value": _value, "render_config": render_config},
)
render_result = result["render_value_result"]
if render_result.data_type_name != "render_value_result":
raise Exception(
f"Invalid result type for render operation: {render_result.data_type_name}"
)
return render_result.data
retrieve_job_result(self, job_id)
¶Retrieve the result(s) of the specified job.
Source code in kiara/interfaces/python_api/__init__.py
def retrieve_job_result(self, job_id: Union[str, uuid.UUID]) -> ValueMap:
"""Retrieve the result(s) of the specified job."""
if isinstance(job_id, str):
job_id = uuid.UUID(job_id)
result = self.context.job_registry.retrieve_result(job_id=job_id)
return result
retrieve_module_type_info(self, module_type)
¶Retrieve information about a specific module type.
This can be used to retrieve information like module documentation and configuration options.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
module_type |
str |
the registered name of the module |
required |
Returns:
| Type | Description |
|---|---|
ModuleTypeInfo |
an object containing all information about a module type |
Source code in kiara/interfaces/python_api/__init__.py
def retrieve_module_type_info(self, module_type: str) -> ModuleTypeInfo:
"""Retrieve information about a specific module type.
This can be used to retrieve information like module documentation and configuration options.
Arguments:
module_type: the registered name of the module
Returns:
an object containing all information about a module type
"""
m_cls = self._kiara.module_registry.get_module_class(module_type)
info = ModuleTypeInfo.create_from_type_class(kiara=self.context, type_cls=m_cls)
return info
retrieve_module_types_info(self, filter=None)
¶Retrieve information for all available module types (or a filtered subset thereof).
A module type is Python class that inherits from KiaraModule, and is the basic building block for processing pipelines. Module types are not used directly by users, Operations are. Operations are instantiated modules (meaning: the module & some (optional) configuration).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filter |
Union[NoneType, str, Iterable[str]] |
a string (or list of string) the returned module names have to match (all filters in case of list) |
None |
Returns:
| Type | Description |
|---|---|
ModuleTypesInfo |
a mapping object containing module names as keys, and information about the modules as values |
Source code in kiara/interfaces/python_api/__init__.py
def retrieve_module_types_info(
self, filter: Union[None, str, Iterable[str]] = None
) -> ModuleTypesInfo:
"""Retrieve information for all available module types (or a filtered subset thereof).
A module type is Python class that inherits from [KiaraModule][kiara.modules.KiaraModule], and is the basic
building block for processing pipelines. Module types are not used directly by users, Operations are. Operations
are instantiated modules (meaning: the module & some (optional) configuration).
Arguments:
filter: a string (or list of string) the returned module names have to match (all filters in case of list)
Returns:
a mapping object containing module names as keys, and information about the modules as values
"""
if filter:
title = f"Filtered modules: {filter}"
module_types_names: Iterable[str] = []
for m in self._kiara.module_registry.get_module_type_names():
match = True
for f in filter:
if f.lower() not in m.lower():
match = False
break
if match:
module_types_names.append(m) # type: ignore
else:
title = "All modules"
module_types_names = self._kiara.module_registry.get_module_type_names()
module_types = {
n: self._kiara.module_registry.get_module_class(n)
for n in module_types_names
}
module_types_info = ModuleTypesInfo.create_from_type_items( # type: ignore
kiara=self.context, group_title=title, **module_types
)
return module_types_info # type: ignore
run_job(self, operation, inputs, operation_config=None)
¶Run a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result.
This is a convenience method that auto-detects what is meant by the 'operation' string input argument.
In general, try to avoid this method and use 'queue_job', 'get_job' and 'retrieve_job_result' manually instead, since this is a blocking operation.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
operation |
Union[str, pathlib.Path, kiara.models.module.manifest.Manifest] |
a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found).. |
required |
inputs |
Mapping[str, Any] |
the operation inputs |
required |
operation_config |
Optional[Mapping[str, Any]] |
the (optional) module config in case 'operation' is a module name |
None |
Returns:
| Type | Description |
|---|---|
ValueMap |
the job result value map |
Source code in kiara/interfaces/python_api/__init__.py
def run_job(
self,
operation: Union[str, Path, Manifest],
inputs: Mapping[str, Any],
operation_config: Union[None, Mapping[str, Any]] = None,
) -> ValueMap:
"""Run a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result.
This is a convenience method that auto-detects what is meant by the 'operation' string input argument.
In general, try to avoid this method and use 'queue_job', 'get_job' and 'retrieve_job_result' manually instead,
since this is a blocking operation.
Arguments:
operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found)..
inputs: the operation inputs
operation_config: the (optional) module config in case 'operation' is a module name
Returns:
the job result value map
"""
job_id = self.queue_job(
operation=operation, inputs=inputs, operation_config=operation_config
)
return self.context.job_registry.retrieve_result(job_id=job_id)
run_manifest(self, manifest, inputs=None)
¶Run a job using the provided manifest to describe the module and config that should be executed.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
manifest |
Manifest |
the manifest |
required |
inputs |
Optional[Mapping[str, Any]] |
the job inputs (can be either references to values, or raw inputs |
None |
Returns:
| Type | Description |
|---|---|
ValueMap |
a result value map instance |
Source code in kiara/interfaces/python_api/__init__.py
def run_manifest(
self, manifest: Manifest, inputs: Union[None, Mapping[str, Any]] = None
) -> ValueMap:
"""Run a job using the provided manifest to describe the module and config that should be executed.
Arguments:
manifest: the manifest
inputs: the job inputs (can be either references to values, or raw inputs
Returns:
a result value map instance
"""
job_id = self.queue_manifest(manifest=manifest, inputs=inputs)
return self.context.job_registry.retrieve_result(job_id=job_id)
store_value(self, value, aliases)
¶Store the specified value in the (default) value store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Union[str, uuid.UUID, kiara.registries.data.ValueLink] |
the value (or a reference to it) |
required |
aliases |
Union[str, Iterable[str]] |
(Optional) aliases for the value |
required |
Source code in kiara/interfaces/python_api/__init__.py
def store_value(
self,
value: Union[str, uuid.UUID, ValueLink],
aliases: Union[str, Iterable[str], None],
):
"""Store the specified value in the (default) value store.
Arguments:
value: the value (or a reference to it)
aliases: (Optional) aliases for the value
"""
if isinstance(aliases, str):
aliases = [aliases]
value_obj = self.get_value(value)
persisted_data: Union[None, PersistedData] = None
try:
persisted_data = self.context.data_registry.store_value(value=value_obj)
if aliases:
self.context.alias_registry.register_aliases(
value_obj.value_id, *aliases
)
result = StoreValueResult.construct(
value=value_obj,
aliases=sorted(aliases) if aliases else [],
error=None,
persisted_data=persisted_data,
)
except Exception as e:
log_exception(e)
result = StoreValueResult.construct(
value=value_obj,
aliases=sorted(aliases) if aliases else [],
error=str(e),
persisted_data=persisted_data,
)
return result
store_values(self, values, alias_map)
¶Store multiple values into the (default) kiara value store.
Values are identified by unique keys in both input arguments, the alias map references the key that is used in the 'values' argument.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
values |
Mapping[str, Union[str, uuid.UUID, kiara.registries.data.ValueLink]] |
a map of value keys/values |
required |
alias_map |
Mapping[str, Iterable[str]] |
a map of value keys aliases |
required |
Returns:
| Type | Description |
|---|---|
StoreValuesResult |
an object outlining which values (identified by the specified value key) where stored and how |
Source code in kiara/interfaces/python_api/__init__.py
def store_values(
self,
values: Mapping[str, Union[str, uuid.UUID, ValueLink]],
alias_map: Mapping[str, Iterable[str]],
) -> StoreValuesResult:
"""Store multiple values into the (default) kiara value store.
Values are identified by unique keys in both input arguments, the alias map references the key that is used in
the 'values' argument.
Arguments:
values: a map of value keys/values
alias_map: a map of value keys aliases
Returns:
an object outlining which values (identified by the specified value key) where stored and how
"""
result = {}
for field_name, value in values.items():
aliases = alias_map.get(field_name)
value_obj = self.get_value(value)
store_result = self.store_value(value=value_obj, aliases=aliases)
result[field_name] = store_result
return StoreValuesResult.construct(__root__=result)
Modules¶
batch
¶
BatchOperation (BaseModel)
pydantic-model
¶Source code in kiara/interfaces/python_api/batch.py
class BatchOperation(BaseModel):
class Config:
validate_assignment = True
@classmethod
def from_file(
cls,
path: str,
kiara: Union["Kiara", None] = None,
):
data = get_data_from_file(path)
pipeline_id = data.get("pipeline_name", None)
if pipeline_id is None:
name = os.path.basename(path)
if name.endswith(".json"):
name = name[0:-5]
elif name.endswith(".yaml"):
name = name[0:-5]
data["pipeline_name"] = name
return cls.from_config(data=data, kiara=kiara)
@classmethod
def from_config(
cls,
data: Mapping[str, Any],
kiara: Union["Kiara", None],
):
data = dict(data)
inputs = data.pop("inputs", {})
save = data.pop("save", False)
pipeline_id = data.pop("pipeline_name", None)
if pipeline_id is None:
pipeline_id = str(uuid.uuid4())
if kiara is None:
kiara = Kiara.instance()
pipeline_config = PipelineConfig.from_config(
pipeline_name=pipeline_id, data=data, kiara=kiara
)
result = cls(pipeline_config=pipeline_config, inputs=inputs, save=save)
result._kiara = kiara
return result
alias: str = Field(description="The batch name/alias.")
pipeline_config: PipelineConfig = Field(
description="The configuration of the underlying pipeline."
)
inputs: Dict[str, Any] = Field(
description="The (base) inputs to use. Can be augmented before running the operation."
)
save_defaults: Dict[str, List[str]] = Field(
description="Configuration which values to save, under which alias(es).",
default_factory=dict,
)
_kiara: Kiara = PrivateAttr(default=None)
@root_validator(pre=True)
def add_alias(cls, values):
if not values.get("alias", None):
pc = values.get("pipeline_config", None)
if not pc:
raise ValueError("No pipeline config provided.")
if isinstance(pc, PipelineConfig):
alias = pc.pipeline_name
else:
alias = pc.get("pipeline_name", None)
values["alias"] = alias
return values
@validator("save_defaults", always=True, pre=True)
def validate_save(cls, save, values):
alias = values["alias"]
pipeline_config = values["pipeline_config"]
return cls.create_save_aliases(
save=save, alias=alias, pipeline_config=pipeline_config
)
@classmethod
def create_save_aliases(
cls,
save: Union[bool, None, str, Mapping],
alias: str,
pipeline_config: PipelineConfig,
) -> Mapping[str, Any]:
assert isinstance(pipeline_config, PipelineConfig)
if save in [False, None]:
save_new: Dict[str, Any] = {}
elif save is True:
field_names = pipeline_config.structure.pipeline_outputs_schema.keys()
save_new = create_save_config(field_names=field_names, aliases=alias)
elif isinstance(save, str):
field_names = pipeline_config.structure.pipeline_outputs_schema.keys()
save_new = create_save_config(field_names=field_names, aliases=save)
elif isinstance(save, Mapping):
save_new = dict(save)
else:
raise ValueError(
f"Invalid type '{type(save)}' for 'save' attribute: must be None, bool, string or Mapping."
)
return save_new
def run(
self,
inputs: Union[Mapping[str, Any], None] = None,
save: Union[None, bool, str, Mapping[str, Any]] = None,
) -> ValueMap:
pipeline = Pipeline(
structure=self.pipeline_config.structure,
kiara=self._kiara,
)
pipeline_controller = SinglePipelineBatchController(
pipeline=pipeline, job_registry=self._kiara.job_registry
)
run_inputs = dict(self.inputs)
if inputs:
run_inputs.update(inputs)
pipeline.set_pipeline_inputs(inputs=run_inputs)
pipeline_controller.process_pipeline()
result = self._kiara.data_registry.load_values(
pipeline.get_current_pipeline_outputs()
)
if save is not None:
if save is True:
save = self.save_defaults
else:
save = self.__class__.create_save_aliases(
save=save, alias=self.alias, pipeline_config=self.pipeline_config
)
self._kiara.save_values(values=result, alias_map=save)
return result
alias: str
pydantic-field
required
¶The batch name/alias.
inputs: Dict[str, Any]
pydantic-field
required
¶The (base) inputs to use. Can be augmented before running the operation.
pipeline_config: PipelineConfig
pydantic-field
required
¶The configuration of the underlying pipeline.
save_defaults: Dict[str, List[str]]
pydantic-field
¶Configuration which values to save, under which alias(es).
Config
¶Source code in kiara/interfaces/python_api/batch.py
class Config:
validate_assignment = True
add_alias(values)
classmethod
¶Source code in kiara/interfaces/python_api/batch.py
@root_validator(pre=True)
def add_alias(cls, values):
if not values.get("alias", None):
pc = values.get("pipeline_config", None)
if not pc:
raise ValueError("No pipeline config provided.")
if isinstance(pc, PipelineConfig):
alias = pc.pipeline_name
else:
alias = pc.get("pipeline_name", None)
values["alias"] = alias
return values
create_save_aliases(save, alias, pipeline_config)
classmethod
¶Source code in kiara/interfaces/python_api/batch.py
@classmethod
def create_save_aliases(
cls,
save: Union[bool, None, str, Mapping],
alias: str,
pipeline_config: PipelineConfig,
) -> Mapping[str, Any]:
assert isinstance(pipeline_config, PipelineConfig)
if save in [False, None]:
save_new: Dict[str, Any] = {}
elif save is True:
field_names = pipeline_config.structure.pipeline_outputs_schema.keys()
save_new = create_save_config(field_names=field_names, aliases=alias)
elif isinstance(save, str):
field_names = pipeline_config.structure.pipeline_outputs_schema.keys()
save_new = create_save_config(field_names=field_names, aliases=save)
elif isinstance(save, Mapping):
save_new = dict(save)
else:
raise ValueError(
f"Invalid type '{type(save)}' for 'save' attribute: must be None, bool, string or Mapping."
)
return save_new
from_config(data, kiara)
classmethod
¶Source code in kiara/interfaces/python_api/batch.py
@classmethod
def from_config(
cls,
data: Mapping[str, Any],
kiara: Union["Kiara", None],
):
data = dict(data)
inputs = data.pop("inputs", {})
save = data.pop("save", False)
pipeline_id = data.pop("pipeline_name", None)
if pipeline_id is None:
pipeline_id = str(uuid.uuid4())
if kiara is None:
kiara = Kiara.instance()
pipeline_config = PipelineConfig.from_config(
pipeline_name=pipeline_id, data=data, kiara=kiara
)
result = cls(pipeline_config=pipeline_config, inputs=inputs, save=save)
result._kiara = kiara
return result
from_file(path, kiara=None)
classmethod
¶Source code in kiara/interfaces/python_api/batch.py
@classmethod
def from_file(
cls,
path: str,
kiara: Union["Kiara", None] = None,
):
data = get_data_from_file(path)
pipeline_id = data.get("pipeline_name", None)
if pipeline_id is None:
name = os.path.basename(path)
if name.endswith(".json"):
name = name[0:-5]
elif name.endswith(".yaml"):
name = name[0:-5]
data["pipeline_name"] = name
return cls.from_config(data=data, kiara=kiara)
run(self, inputs=None, save=None)
¶Source code in kiara/interfaces/python_api/batch.py
def run(
self,
inputs: Union[Mapping[str, Any], None] = None,
save: Union[None, bool, str, Mapping[str, Any]] = None,
) -> ValueMap:
pipeline = Pipeline(
structure=self.pipeline_config.structure,
kiara=self._kiara,
)
pipeline_controller = SinglePipelineBatchController(
pipeline=pipeline, job_registry=self._kiara.job_registry
)
run_inputs = dict(self.inputs)
if inputs:
run_inputs.update(inputs)
pipeline.set_pipeline_inputs(inputs=run_inputs)
pipeline_controller.process_pipeline()
result = self._kiara.data_registry.load_values(
pipeline.get_current_pipeline_outputs()
)
if save is not None:
if save is True:
save = self.save_defaults
else:
save = self.__class__.create_save_aliases(
save=save, alias=self.alias, pipeline_config=self.pipeline_config
)
self._kiara.save_values(values=result, alias_map=save)
return result
validate_save(save, values)
classmethod
¶Source code in kiara/interfaces/python_api/batch.py
@validator("save_defaults", always=True, pre=True)
def validate_save(cls, save, values):
alias = values["alias"]
pipeline_config = values["pipeline_config"]
return cls.create_save_aliases(
save=save, alias=alias, pipeline_config=pipeline_config
)
models
special
¶info
¶INFO_BASE_CLASS
¶INFO_BASE_INSTANCE_TYPE
¶INFO_ITEM_TYPE
¶RENDER_FIELDS: Dict[str, Dict[str, Any]]
¶
DataTypeClassInfo (TypeInfo)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class DataTypeClassInfo(TypeInfo[Type["DataType"]]):
_kiara_model_id = "info.data_type"
@classmethod
def create_from_type_class(
self, type_cls: Type["DataType"], kiara: Union["Kiara", None] = None
) -> "DataTypeClassInfo":
authors = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
if kiara is not None:
qual_profiles = kiara.type_registry.get_associated_profiles(type_cls._data_type_name) # type: ignore
lineage = kiara.type_registry.get_type_lineage(type_cls._data_type_name) # type: ignore
else:
qual_profiles = None
lineage = None
try:
result = DataTypeClassInfo.construct(
type_name=type_cls._data_type_name, # type: ignore
python_class=PythonClass.from_class(type_cls),
value_cls=PythonClass.from_class(type_cls.python_class()),
data_type_config_cls=PythonClass.from_class(
type_cls.data_type_config_class()
),
lineage=lineage, # type: ignore
qualifier_profiles=qual_profiles,
documentation=doc,
authors=authors,
context=properties_md,
)
except Exception as e:
if isinstance(
e, TypeError
) and "missing 1 required positional argument: 'cls'" in str(e):
raise Exception(
f"Invalid implementation of TypeValue subclass '{type_cls.__name__}': 'python_class' method must be marked as a '@classmethod'. This is a bug."
)
raise e
result._kiara = kiara
return result
@classmethod
def base_class(self) -> Type["DataType"]:
from kiara.data_types import DataType
return DataType
@classmethod
def category_name(cls) -> str:
return "data_type"
value_cls: PythonClass = Field(description="The python class of the value itself.")
data_type_config_cls: PythonClass = Field(
description="The python class holding the schema for configuring this type."
)
lineage: Union[List[str], None] = Field(description="This types lineage.")
qualifier_profiles: Union[Mapping[str, Mapping[str, Any]], None] = Field(
description="A map of qualifier profiles for this data types."
)
_kiara: Union["Kiara", None] = PrivateAttr(default=None)
def _retrieve_id(self) -> str:
return self.type_name
def _retrieve_data_to_hash(self) -> Any:
return self.type_name
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if self.lineage:
table.add_row("lineage", "\n".join(self.lineage[0:]))
else:
table.add_row("lineage", "-- n/a --")
if self.qualifier_profiles:
qual_table = Table(show_header=False, box=box.SIMPLE)
qual_table.add_column("name")
qual_table.add_column("config")
for name, details in self.qualifier_profiles.items():
json_details = orjson_dumps(details, option=orjson.OPT_INDENT_2)
qual_table.add_row(
name, Syntax(json_details, "json", background_color="default")
)
table.add_row("qualifier profile(s)", qual_table)
else:
table.add_row("qualifier profile(s)", "-- n/a --")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
table.add_row("Python class", self.python_class.create_renderable())
table.add_row("Config class", self.data_type_config_cls.create_renderable())
table.add_row("Value class", self.value_cls.create_renderable())
return table
data_type_config_cls: PythonClass
pydantic-field
required
¶The python class holding the schema for configuring this type.
lineage: List[str]
pydantic-field
¶This types lineage.
qualifier_profiles: Mapping[str, Mapping[str, Any]]
pydantic-field
¶A map of qualifier profiles for this data types.
value_cls: PythonClass
pydantic-field
required
¶The python class of the value itself.
base_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_class(self) -> Type["DataType"]:
from kiara.data_types import DataType
return DataType
category_name()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def category_name(cls) -> str:
return "data_type"
create_from_type_class(type_cls, kiara=None)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_type_class(
self, type_cls: Type["DataType"], kiara: Union["Kiara", None] = None
) -> "DataTypeClassInfo":
authors = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
if kiara is not None:
qual_profiles = kiara.type_registry.get_associated_profiles(type_cls._data_type_name) # type: ignore
lineage = kiara.type_registry.get_type_lineage(type_cls._data_type_name) # type: ignore
else:
qual_profiles = None
lineage = None
try:
result = DataTypeClassInfo.construct(
type_name=type_cls._data_type_name, # type: ignore
python_class=PythonClass.from_class(type_cls),
value_cls=PythonClass.from_class(type_cls.python_class()),
data_type_config_cls=PythonClass.from_class(
type_cls.data_type_config_class()
),
lineage=lineage, # type: ignore
qualifier_profiles=qual_profiles,
documentation=doc,
authors=authors,
context=properties_md,
)
except Exception as e:
if isinstance(
e, TypeError
) and "missing 1 required positional argument: 'cls'" in str(e):
raise Exception(
f"Invalid implementation of TypeValue subclass '{type_cls.__name__}': 'python_class' method must be marked as a '@classmethod'. This is a bug."
)
raise e
result._kiara = kiara
return result
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if self.lineage:
table.add_row("lineage", "\n".join(self.lineage[0:]))
else:
table.add_row("lineage", "-- n/a --")
if self.qualifier_profiles:
qual_table = Table(show_header=False, box=box.SIMPLE)
qual_table.add_column("name")
qual_table.add_column("config")
for name, details in self.qualifier_profiles.items():
json_details = orjson_dumps(details, option=orjson.OPT_INDENT_2)
qual_table.add_row(
name, Syntax(json_details, "json", background_color="default")
)
table.add_row("qualifier profile(s)", qual_table)
else:
table.add_row("qualifier profile(s)", "-- n/a --")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
table.add_row("Python class", self.python_class.create_renderable())
table.add_row("Config class", self.data_type_config_cls.create_renderable())
table.add_row("Value class", self.value_cls.create_renderable())
return table
DataTypeClassesInfo (TypeInfoItemGroup)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class DataTypeClassesInfo(TypeInfoItemGroup):
_kiara_model_id = "info.data_types"
# @classmethod
# def create_from_type_items(
# cls,
# group_title: Union[str, None] = None,
# **items: Type,
# ) -> "TypeInfoModelGroup":
#
# type_infos = {
# k: cls.base_info_class().create_from_type_class(v) for k, v in items.items() # type: ignore
# }
# data_types_info = cls.construct(group_alias=group_title, item_infos=type_infos) # type: ignore
# return data_types_info
#
# @classmethod
# def create_augmented_from_type_items(
# cls,
# kiara: Union["Kiara", None] = None,
# group_alias: Union[str, None] = None,
# **items: Type,
# ) -> "TypeInfoModelGroup":
#
# type_infos = {
# k: cls.base_info_class().create_from_type_class(v, kiara=kiara) for k, v in items.items() # type: ignore
# }
# data_types_info = cls.construct(group_alias=group_alias, item_infos=type_infos) # type: ignore
# data_types_info._kiara = kiara
# return data_types_info
@classmethod
def base_info_class(cls) -> Type[DataTypeClassInfo]:
return DataTypeClassInfo
type_name: Literal["data_type"] = "data_type"
item_infos: Mapping[str, DataTypeClassInfo] = Field( # type: ignore
description="The data_type info instances for each type."
)
# _kiara: Union["Kiara", None] = PrivateAttr(default=None)
def create_renderable(self, **config: Any) -> RenderableType:
full_doc = config.get("full_doc", False)
show_subtypes_inline = config.get("show_qualifier_profiles_inline", True)
show_lineage = config.get("show_type_lineage", True)
show_lines = full_doc or show_subtypes_inline or show_lineage
table = Table(show_header=True, box=box.SIMPLE, show_lines=show_lines)
table.add_column("type name", style="i")
if show_lineage:
table.add_column("type lineage")
if show_subtypes_inline:
table.add_column("(qualifier) profiles")
if full_doc:
table.add_column("documentation")
else:
table.add_column("description")
all_types = self.item_infos.keys()
for type_name in sorted(all_types): # type: ignore
t_md = self.item_infos[type_name] # type: ignore
row: List[Any] = [type_name]
if show_lineage:
if self._kiara is None:
lineage_str = "-- n/a --"
else:
lineage = list(
self._kiara.type_registry.get_type_lineage(type_name)
)
lineage_str = ", ".join(reversed(lineage[1:]))
row.append(lineage_str)
if show_subtypes_inline:
if self._kiara is None:
qual_profiles = "-- n/a --"
else:
qual_p = self._kiara.type_registry.get_associated_profiles(
data_type_name=type_name
).keys()
if qual_p:
qual_profiles = "\n".join(qual_p)
else:
qual_profiles = "-- n/a --"
row.append(qual_profiles)
if full_doc:
md = Markdown(t_md.documentation.full_doc)
else:
md = Markdown(t_md.documentation.description)
row.append(md)
table.add_row(*row)
return table
type_name: Literal['data_type']
pydantic-field
¶base_info_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_info_class(cls) -> Type[DataTypeClassInfo]:
return DataTypeClassInfo
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, **config: Any) -> RenderableType:
full_doc = config.get("full_doc", False)
show_subtypes_inline = config.get("show_qualifier_profiles_inline", True)
show_lineage = config.get("show_type_lineage", True)
show_lines = full_doc or show_subtypes_inline or show_lineage
table = Table(show_header=True, box=box.SIMPLE, show_lines=show_lines)
table.add_column("type name", style="i")
if show_lineage:
table.add_column("type lineage")
if show_subtypes_inline:
table.add_column("(qualifier) profiles")
if full_doc:
table.add_column("documentation")
else:
table.add_column("description")
all_types = self.item_infos.keys()
for type_name in sorted(all_types): # type: ignore
t_md = self.item_infos[type_name] # type: ignore
row: List[Any] = [type_name]
if show_lineage:
if self._kiara is None:
lineage_str = "-- n/a --"
else:
lineage = list(
self._kiara.type_registry.get_type_lineage(type_name)
)
lineage_str = ", ".join(reversed(lineage[1:]))
row.append(lineage_str)
if show_subtypes_inline:
if self._kiara is None:
qual_profiles = "-- n/a --"
else:
qual_p = self._kiara.type_registry.get_associated_profiles(
data_type_name=type_name
).keys()
if qual_p:
qual_profiles = "\n".join(qual_p)
else:
qual_profiles = "-- n/a --"
row.append(qual_profiles)
if full_doc:
md = Markdown(t_md.documentation.full_doc)
else:
md = Markdown(t_md.documentation.description)
row.append(md)
table.add_row(*row)
return table
FieldInfo (BaseModel)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class FieldInfo(BaseModel):
field_name: str = Field(description="The field name.")
field_schema: ValueSchema = Field(description="The schema of the field.")
data_type_info: DataTypeInfo = Field(
description="Information about the data type instance of the associated value."
)
value_required: bool = Field(
description="Whether user input is required (meaning: 'optional' is False, and no default set)."
)
data_type_info: DataTypeInfo
pydantic-field
required
¶Information about the data type instance of the associated value.
field_name: str
pydantic-field
required
¶The field name.
field_schema: ValueSchema
pydantic-field
required
¶The schema of the field.
value_required: bool
pydantic-field
required
¶Whether user input is required (meaning: 'optional' is False, and no default set).
InfoItemGroup (KiaraModel, Generic)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class InfoItemGroup(KiaraModel, Generic[INFO_ITEM_TYPE]):
@classmethod
@abc.abstractmethod
def base_info_class(cls) -> Type[INFO_ITEM_TYPE]:
pass
@classmethod
def create_from_instances(
cls,
kiara: "Kiara",
instances: Mapping[str, INFO_BASE_INSTANCE_TYPE],
**kwargs: Any,
) -> "InfoItemGroup[INFO_ITEM_TYPE]":
info_cls = cls.base_info_class()
items = {}
for k, v in instances.items():
items[k] = info_cls.create_from_instance(kiara=kiara, instance=v, **kwargs)
group_title = kwargs.pop("group_title", None)
result = cls(group_title=group_title, item_infos=items)
result._kiara = kiara
return result
group_title: Union[str, None] = Field(description="The group alias.", default=None)
item_infos: Mapping[str, INFO_ITEM_TYPE] = Field(description="The info items.")
_kiara: Union["Kiara", None] = PrivateAttr(default=None)
def _retrieve_subcomponent_keys(self) -> Iterable[str]:
return self.item_infos.keys()
def _retrieve_data_to_hash(self) -> Any:
return {"type_name": self.__class__._kiara_model_name, "included_types": list(self.item_infos.keys())} # type: ignore
def create_renderable(self, **config: Any) -> RenderableType:
full_doc = config.get("full_doc", False)
table = Table(show_header=True, box=box.SIMPLE, show_lines=full_doc)
table.add_column("Name", style="i")
table.add_column("Description")
for type_name in sorted(self.item_infos.keys()):
t_md = self.item_infos[type_name]
if full_doc:
md = Markdown(t_md.documentation.full_doc)
else:
md = Markdown(t_md.documentation.description)
table.add_row(type_name, md)
return table
def __getitem__(self, item: str) -> INFO_ITEM_TYPE:
return self.item_infos[item]
# def __iter__(self):
# return self.item_infos.__iter__()
def __len__(self):
return len(self.item_infos)
group_title: str
pydantic-field
¶The group alias.
item_infos: Mapping[str, ~INFO_ITEM_TYPE]
pydantic-field
required
¶The info items.
base_info_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
@abc.abstractmethod
def base_info_class(cls) -> Type[INFO_ITEM_TYPE]:
pass
create_from_instances(kiara, instances, **kwargs)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_instances(
cls,
kiara: "Kiara",
instances: Mapping[str, INFO_BASE_INSTANCE_TYPE],
**kwargs: Any,
) -> "InfoItemGroup[INFO_ITEM_TYPE]":
info_cls = cls.base_info_class()
items = {}
for k, v in instances.items():
items[k] = info_cls.create_from_instance(kiara=kiara, instance=v, **kwargs)
group_title = kwargs.pop("group_title", None)
result = cls(group_title=group_title, item_infos=items)
result._kiara = kiara
return result
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, **config: Any) -> RenderableType:
full_doc = config.get("full_doc", False)
table = Table(show_header=True, box=box.SIMPLE, show_lines=full_doc)
table.add_column("Name", style="i")
table.add_column("Description")
for type_name in sorted(self.item_infos.keys()):
t_md = self.item_infos[type_name]
if full_doc:
md = Markdown(t_md.documentation.full_doc)
else:
md = Markdown(t_md.documentation.description)
table.add_row(type_name, md)
return table
ItemInfo (KiaraModel, Generic)
pydantic-model
¶Base class that holds/manages information about an item within kiara.
Source code in kiara/interfaces/python_api/models/info.py
class ItemInfo(KiaraModel, Generic[INFO_BASE_INSTANCE_TYPE]):
"""Base class that holds/manages information about an item within kiara."""
@classmethod
@abc.abstractmethod
def base_instance_class(cls) -> Type[INFO_BASE_INSTANCE_TYPE]:
pass
@classmethod
@abc.abstractmethod
def create_from_instance(
cls, kiara: "Kiara", instance: INFO_BASE_INSTANCE_TYPE, **kwargs
):
pass
@validator("documentation", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
type_name: str = Field(description="The registered name for this item type.")
documentation: DocumentationMetadataModel = Field(
description="Documentation for the item."
)
authors: AuthorsMetadataModel = Field(
description="Information about authorship for the item."
)
context: ContextMetadataModel = Field(
description="Generic properties of this item (description, tags, labels, references, ...)."
)
def _retrieve_id(self) -> str:
return self.type_name
def _retrieve_data_to_hash(self) -> Any:
return self.type_name
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
if hasattr(self, "python_class"):
table.add_row("Python class", self.python_class.create_renderable()) # type: ignore
return table
authors: AuthorsMetadataModel
pydantic-field
required
¶Information about authorship for the item.
context: ContextMetadataModel
pydantic-field
required
¶Generic properties of this item (description, tags, labels, references, ...).
documentation: DocumentationMetadataModel
pydantic-field
required
¶Documentation for the item.
type_name: str
pydantic-field
required
¶The registered name for this item type.
base_instance_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
@abc.abstractmethod
def base_instance_class(cls) -> Type[INFO_BASE_INSTANCE_TYPE]:
pass
create_from_instance(kiara, instance, **kwargs)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
@abc.abstractmethod
def create_from_instance(
cls, kiara: "Kiara", instance: INFO_BASE_INSTANCE_TYPE, **kwargs
):
pass
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
if hasattr(self, "python_class"):
table.add_row("Python class", self.python_class.create_renderable()) # type: ignore
return table
validate_doc(value)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@validator("documentation", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
KiaraModelClassesInfo (TypeInfoItemGroup)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class KiaraModelClassesInfo(TypeInfoItemGroup):
_kiara_model_id = "info.kiara_models"
@classmethod
def find_kiara_models(
cls, alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> "KiaraModelClassesInfo":
models = find_all_kiara_model_classes()
# we don't need the kiara instance, this is just to satisfy mypy
kiara: Kiara = None # type: ignore
group: KiaraModelClassesInfo = KiaraModelClassesInfo.create_from_type_items(kiara=kiara, group_title=alias, **models) # type: ignore
if only_for_package:
temp = {}
for key, info in group.item_infos.items():
if info.context.labels.get("package") == only_for_package:
temp[key] = info
group = KiaraModelClassesInfo.construct(
group_id=group.instance_id, group_alias=group.group_alias, item_infos=temp # type: ignore
)
return group
@classmethod
def base_info_class(cls) -> Type[KiaraModelTypeInfo]:
return KiaraModelTypeInfo # type: ignore
type_name: Literal["kiara_model"] = "kiara_model"
item_infos: Mapping[str, KiaraModelTypeInfo] = Field( # type: ignore
description="The value metadata info instances for each type."
)
type_name: Literal['kiara_model']
pydantic-field
¶base_info_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_info_class(cls) -> Type[KiaraModelTypeInfo]:
return KiaraModelTypeInfo # type: ignore
find_kiara_models(alias=None, only_for_package=None)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def find_kiara_models(
cls, alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> "KiaraModelClassesInfo":
models = find_all_kiara_model_classes()
# we don't need the kiara instance, this is just to satisfy mypy
kiara: Kiara = None # type: ignore
group: KiaraModelClassesInfo = KiaraModelClassesInfo.create_from_type_items(kiara=kiara, group_title=alias, **models) # type: ignore
if only_for_package:
temp = {}
for key, info in group.item_infos.items():
if info.context.labels.get("package") == only_for_package:
temp[key] = info
group = KiaraModelClassesInfo.construct(
group_id=group.instance_id, group_alias=group.group_alias, item_infos=temp # type: ignore
)
return group
KiaraModelTypeInfo (TypeInfo)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class KiaraModelTypeInfo(TypeInfo[Type[KiaraModel]]):
_kiara_model_id = "info.kiara_model"
@classmethod
def create_from_type_class(
self, type_cls: Type[KiaraModel], kiara: "Kiara"
) -> "KiaraModelTypeInfo":
authors_md = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
python_class = PythonClass.from_class(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
type_name = type_cls._kiara_model_id # type: ignore
schema = type_cls.schema()
return KiaraModelTypeInfo.construct(
type_name=type_name,
documentation=doc,
authors=authors_md,
context=properties_md,
python_class=python_class,
metadata_schema=schema,
)
metadata_schema: Dict[str, Any] = Field(
description="The (json) schema for this model data."
)
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
include_schema = config.get("include_schema", False)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
if hasattr(self, "python_class"):
table.add_row("Python class", self.python_class.create_renderable())
if include_schema:
schema = Syntax(
orjson_dumps(self.metadata_schema, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
table.add_row("metadata_schema", schema)
return table
metadata_schema: Dict[str, Any]
pydantic-field
required
¶The (json) schema for this model data.
create_from_type_class(type_cls, kiara)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_type_class(
self, type_cls: Type[KiaraModel], kiara: "Kiara"
) -> "KiaraModelTypeInfo":
authors_md = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
python_class = PythonClass.from_class(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
type_name = type_cls._kiara_model_id # type: ignore
schema = type_cls.schema()
return KiaraModelTypeInfo.construct(
type_name=type_name,
documentation=doc,
authors=authors_md,
context=properties_md,
python_class=python_class,
metadata_schema=schema,
)
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
include_schema = config.get("include_schema", False)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
if hasattr(self, "python_class"):
table.add_row("Python class", self.python_class.create_renderable())
if include_schema:
schema = Syntax(
orjson_dumps(self.metadata_schema, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
table.add_row("metadata_schema", schema)
return table
KiaraModuleConfigMetadata (KiaraModel)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class KiaraModuleConfigMetadata(KiaraModel):
_kiara_model_id = "metadata.module_config"
@classmethod
def from_config_class(
cls,
config_cls: Type[KiaraModuleConfig],
):
flat_models = get_flat_models_from_model(config_cls)
model_name_map = get_model_name_map(flat_models)
m_schema, _, _ = model_process_schema(config_cls, model_name_map=model_name_map)
fields = m_schema["properties"]
config_values = {}
for field_name, details in fields.items():
type_str = "-- n/a --"
if "type" in details.keys():
type_str = details["type"]
desc = details.get("description", DEFAULT_NO_DESC_VALUE)
default = config_cls.__fields__[field_name].default
if default is None:
if callable(config_cls.__fields__[field_name].default_factory):
default = config_cls.__fields__[field_name].default_factory() # type: ignore
req = config_cls.__fields__[field_name].required
config_values[field_name] = ValueTypeAndDescription(
description=desc, type=type_str, value_default=default, required=req
)
python_cls = PythonClass.from_class(config_cls)
return KiaraModuleConfigMetadata(
python_class=python_cls, config_values=config_values
)
python_class: PythonClass = Field(description="The config model python class.")
config_values: Dict[str, ValueTypeAndDescription] = Field(
description="The available configuration values."
)
def _retrieve_id(self) -> str:
return self.python_class.full_name
def _retrieve_data_to_hash(self) -> Any:
return self.python_class.full_name
config_values: Dict[str, kiara.interfaces.python_api.models.info.ValueTypeAndDescription]
pydantic-field
required
¶The available configuration values.
python_class: PythonClass
pydantic-field
required
¶The config model python class.
from_config_class(config_cls)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def from_config_class(
cls,
config_cls: Type[KiaraModuleConfig],
):
flat_models = get_flat_models_from_model(config_cls)
model_name_map = get_model_name_map(flat_models)
m_schema, _, _ = model_process_schema(config_cls, model_name_map=model_name_map)
fields = m_schema["properties"]
config_values = {}
for field_name, details in fields.items():
type_str = "-- n/a --"
if "type" in details.keys():
type_str = details["type"]
desc = details.get("description", DEFAULT_NO_DESC_VALUE)
default = config_cls.__fields__[field_name].default
if default is None:
if callable(config_cls.__fields__[field_name].default_factory):
default = config_cls.__fields__[field_name].default_factory() # type: ignore
req = config_cls.__fields__[field_name].required
config_values[field_name] = ValueTypeAndDescription(
description=desc, type=type_str, value_default=default, required=req
)
python_cls = PythonClass.from_class(config_cls)
return KiaraModuleConfigMetadata(
python_class=python_cls, config_values=config_values
)
ModuleTypeInfo (TypeInfo)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class ModuleTypeInfo(TypeInfo[Type["KiaraModule"]]):
_kiara_model_id = "info.kiara_module_type"
@classmethod
def create_from_type_class(cls, type_cls: Type["KiaraModule"], kiara: "Kiara") -> "ModuleTypeInfo": # type: ignore
module_attrs = cls.extract_module_attributes(module_cls=type_cls)
return cls.construct(**module_attrs)
@classmethod
def base_class(self) -> Type["KiaraModule"]:
from kiara.modules import KiaraModule
return KiaraModule
@classmethod
def category_name(cls) -> str:
return "module"
@classmethod
def extract_module_attributes(
self, module_cls: Type["KiaraModule"]
) -> Dict[str, Any]:
if not hasattr(module_cls, "process"):
raise Exception(f"Module class '{module_cls}' misses 'process' method.")
proc_src = textwrap.dedent(inspect.getsource(module_cls.process)) # type: ignore
authors_md = AuthorsMetadataModel.from_class(module_cls)
doc = DocumentationMetadataModel.from_class_doc(module_cls)
python_class = PythonClass.from_class(module_cls)
properties_md = ContextMetadataModel.from_class(module_cls)
config = KiaraModuleConfigMetadata.from_config_class(module_cls._config_cls)
return {
"type_name": module_cls._module_type_name, # type: ignore
"documentation": doc,
"authors": authors_md,
"context": properties_md,
"python_class": python_class,
"config": config,
"process_src": proc_src,
}
process_src: str = Field(
description="The source code of the process method of the module."
)
def create_renderable(self, **config: Any) -> RenderableType:
include_config_schema = config.get("include_config_schema", True)
include_src = config.get("include_src", True)
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
if include_config_schema:
config_cls = self.python_class.get_class()._config_cls # type: ignore
from kiara.utils.output import create_table_from_base_model_cls
table.add_row(
"Module config schema", create_table_from_base_model_cls(config_cls)
)
table.add_row("Python class", self.python_class.create_renderable())
if include_src:
_config = Syntax(self.process_src, "python", background_color="default")
table.add_row("Processing source code", Panel(_config, box=box.HORIZONTALS))
return table
process_src: str
pydantic-field
required
¶The source code of the process method of the module.
base_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_class(self) -> Type["KiaraModule"]:
from kiara.modules import KiaraModule
return KiaraModule
category_name()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def category_name(cls) -> str:
return "module"
create_from_type_class(type_cls, kiara)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_type_class(cls, type_cls: Type["KiaraModule"], kiara: "Kiara") -> "ModuleTypeInfo": # type: ignore
module_attrs = cls.extract_module_attributes(module_cls=type_cls)
return cls.construct(**module_attrs)
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, **config: Any) -> RenderableType:
include_config_schema = config.get("include_config_schema", True)
include_src = config.get("include_src", True)
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
if include_config_schema:
config_cls = self.python_class.get_class()._config_cls # type: ignore
from kiara.utils.output import create_table_from_base_model_cls
table.add_row(
"Module config schema", create_table_from_base_model_cls(config_cls)
)
table.add_row("Python class", self.python_class.create_renderable())
if include_src:
_config = Syntax(self.process_src, "python", background_color="default")
table.add_row("Processing source code", Panel(_config, box=box.HORIZONTALS))
return table
extract_module_attributes(module_cls)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def extract_module_attributes(
self, module_cls: Type["KiaraModule"]
) -> Dict[str, Any]:
if not hasattr(module_cls, "process"):
raise Exception(f"Module class '{module_cls}' misses 'process' method.")
proc_src = textwrap.dedent(inspect.getsource(module_cls.process)) # type: ignore
authors_md = AuthorsMetadataModel.from_class(module_cls)
doc = DocumentationMetadataModel.from_class_doc(module_cls)
python_class = PythonClass.from_class(module_cls)
properties_md = ContextMetadataModel.from_class(module_cls)
config = KiaraModuleConfigMetadata.from_config_class(module_cls._config_cls)
return {
"type_name": module_cls._module_type_name, # type: ignore
"documentation": doc,
"authors": authors_md,
"context": properties_md,
"python_class": python_class,
"config": config,
"process_src": proc_src,
}
ModuleTypesInfo (TypeInfoItemGroup)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class ModuleTypesInfo(TypeInfoItemGroup):
_kiara_model_id = "info.module_types"
@classmethod
def base_info_class(cls) -> Type[TypeInfo]:
return ModuleTypeInfo
type_name: Literal["module_type"] = "module_type"
item_infos: Mapping[str, ModuleTypeInfo] = Field( # type: ignore
description="The module type info instances for each type."
)
OperationGroupInfo (InfoItemGroup)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class OperationGroupInfo(InfoItemGroup):
_kiara_model_id = "info.operations"
@classmethod
def base_info_class(cls) -> Type[ItemInfo]:
return OperationInfo
@classmethod
def create_from_operations(
cls, kiara: "Kiara", group_title: Union[str, None] = None, **items: Operation
) -> "OperationGroupInfo":
op_infos = {
k: OperationInfo.create_from_operation(kiara=kiara, operation=v)
for k, v in items.items()
}
op_group_info = cls.construct(group_title=group_title, item_infos=op_infos)
return op_group_info
# type_name: Literal["operation_type"] = "operation_type"
item_infos: Mapping[str, OperationInfo] = Field(
description="The operation info instances for each type."
)
def create_renderable(self, **config: Any) -> RenderableType:
by_type = config.get("by_type", False)
if by_type:
return self._create_renderable_by_type(**config)
else:
return self._create_renderable_list(**config)
def _create_renderable_list(self, **config):
include_internal_operations = config.get("include_internal_operations", True)
full_doc = config.get("full_doc", False)
filter = config.get("filter", [])
table = Table(box=box.SIMPLE, show_header=True)
table.add_column("Id", no_wrap=True, style="i")
table.add_column("Type(s)", style="green")
table.add_column("Description")
for op_id, op_info in self.item_infos.items():
if (
not include_internal_operations
and op_info.operation.operation_details.is_internal_operation
):
continue
types = op_info.operation_types
if "custom_module" in types:
types.remove("custom_module")
desc_str = op_info.documentation.description
if full_doc:
desc = Markdown(op_info.documentation.full_doc)
else:
desc = Markdown(op_info.documentation.description)
if filter:
match = True
for f in filter:
if (
f.lower() not in op_id.lower()
and f.lower() not in desc_str.lower()
):
match = False
break
if match:
table.add_row(op_id, ", ".join(types), desc)
else:
table.add_row(op_id, ", ".join(types), desc)
return table
def _create_renderable_by_type(self, **config):
include_internal_operations = config.get("include_internal_operations", True)
full_doc = config.get("full_doc", False)
filter = config.get("filter", [])
by_type = {}
for op_id, op in self.item_infos.items():
if filter:
match = True
for f in filter:
if (
f.lower() not in op_id.lower()
and f.lower() not in op.documentation.description.lower()
):
match = False
break
if not match:
continue
for op_type in op.operation_types:
by_type.setdefault(op_type, {})[op_id] = op
table = Table(box=box.SIMPLE, show_header=True)
table.add_column("Type", no_wrap=True, style="b green")
table.add_column("Id", no_wrap=True, style="i")
if full_doc:
table.add_column("Documentation", no_wrap=False, style="i")
else:
table.add_column("Description", no_wrap=False, style="i")
for operation_name in sorted(by_type.keys()):
# if operation_name == "custom_module":
# continue
first_line_value = True
op_infos = by_type[operation_name]
for op_id in sorted(op_infos.keys()):
op_info: OperationInfo = op_infos[op_id]
if (
not include_internal_operations
and op_info.operation.operation_details.is_internal_operation
):
continue
if full_doc:
desc = Markdown(op_info.documentation.full_doc)
else:
desc = Markdown(op_info.documentation.description)
row = []
if first_line_value:
row.append(operation_name)
else:
row.append("")
row.append(op_id)
row.append(desc)
table.add_row(*row)
first_line_value = False
return table
base_info_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_info_class(cls) -> Type[ItemInfo]:
return OperationInfo
create_from_operations(kiara, group_title=None, **items)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_operations(
cls, kiara: "Kiara", group_title: Union[str, None] = None, **items: Operation
) -> "OperationGroupInfo":
op_infos = {
k: OperationInfo.create_from_operation(kiara=kiara, operation=v)
for k, v in items.items()
}
op_group_info = cls.construct(group_title=group_title, item_infos=op_infos)
return op_group_info
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, **config: Any) -> RenderableType:
by_type = config.get("by_type", False)
if by_type:
return self._create_renderable_by_type(**config)
else:
return self._create_renderable_list(**config)
OperationInfo (ItemInfo)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class OperationInfo(ItemInfo):
_kiara_model_id = "info.operation"
@classmethod
def base_instance_class(cls) -> Type[Operation]:
return Operation
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: Operation, **kwargs):
return cls.create_from_operation(kiara=kiara, operation=instance)
@classmethod
def create_from_operation(cls, kiara: "Kiara", operation: Operation):
module = operation.module
module_cls = module.__class__
authors_md = AuthorsMetadataModel.from_class(module_cls)
properties_md = ContextMetadataModel.from_class(module_cls)
op_types = kiara.operation_registry.find_all_operation_types(
operation_id=operation.operation_id
)
input_fields = {}
for field_name, schema in operation.inputs_schema.items():
dt = kiara.type_registry.get_data_type_instance(
type_name=schema.type, type_config=schema.type_config
)
dt_info = FieldInfo.construct(
field_name=field_name,
field_schema=schema,
data_type_info=dt.info,
value_required=schema.is_required(),
)
input_fields[field_name] = dt_info
output_fields = {}
for field_name, schema in operation.outputs_schema.items():
dt = kiara.type_registry.get_data_type_instance(
type_name=schema.type, type_config=schema.type_config
)
dt_info = FieldInfo.construct(
field_name=field_name,
field_schema=schema,
data_type_info=dt.info,
value_required=schema.is_required(),
)
output_fields[field_name] = dt_info
op_info = OperationInfo.construct(
type_name=operation.operation_id,
operation_types=list(op_types),
input_fields=input_fields,
output_fields=output_fields,
operation=operation,
documentation=operation.doc,
authors=authors_md,
context=properties_md,
)
return op_info
@classmethod
def category_name(cls) -> str:
return "operation"
operation: Operation = Field(description="The operation instance.")
operation_types: List[str] = Field(
description="The operation types this operation belongs to."
)
input_fields: Mapping[str, FieldInfo] = Field(
description="The inputs schema for this operation."
)
output_fields: Mapping[str, FieldInfo] = Field(
description="The outputs schema for this operation."
)
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable(**config))
table.add_row("Context", self.context.create_renderable(**config))
table.add_row("Operation details", self.operation.create_renderable(**config))
return table
input_fields: Mapping[str, kiara.interfaces.python_api.models.info.FieldInfo]
pydantic-field
required
¶The inputs schema for this operation.
operation: Operation
pydantic-field
required
¶The operation instance.
operation_types: List[str]
pydantic-field
required
¶The operation types this operation belongs to.
output_fields: Mapping[str, kiara.interfaces.python_api.models.info.FieldInfo]
pydantic-field
required
¶The outputs schema for this operation.
base_instance_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_instance_class(cls) -> Type[Operation]:
return Operation
category_name()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def category_name(cls) -> str:
return "operation"
create_from_instance(kiara, instance, **kwargs)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: Operation, **kwargs):
return cls.create_from_operation(kiara=kiara, operation=instance)
create_from_operation(kiara, operation)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_operation(cls, kiara: "Kiara", operation: Operation):
module = operation.module
module_cls = module.__class__
authors_md = AuthorsMetadataModel.from_class(module_cls)
properties_md = ContextMetadataModel.from_class(module_cls)
op_types = kiara.operation_registry.find_all_operation_types(
operation_id=operation.operation_id
)
input_fields = {}
for field_name, schema in operation.inputs_schema.items():
dt = kiara.type_registry.get_data_type_instance(
type_name=schema.type, type_config=schema.type_config
)
dt_info = FieldInfo.construct(
field_name=field_name,
field_schema=schema,
data_type_info=dt.info,
value_required=schema.is_required(),
)
input_fields[field_name] = dt_info
output_fields = {}
for field_name, schema in operation.outputs_schema.items():
dt = kiara.type_registry.get_data_type_instance(
type_name=schema.type, type_config=schema.type_config
)
dt_info = FieldInfo.construct(
field_name=field_name,
field_schema=schema,
data_type_info=dt.info,
value_required=schema.is_required(),
)
output_fields[field_name] = dt_info
op_info = OperationInfo.construct(
type_name=operation.operation_id,
operation_types=list(op_types),
input_fields=input_fields,
output_fields=output_fields,
operation=operation,
documentation=operation.doc,
authors=authors_md,
context=properties_md,
)
return op_info
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable(**config))
table.add_row("Context", self.context.create_renderable(**config))
table.add_row("Operation details", self.operation.create_renderable(**config))
return table
OperationTypeClassesInfo (TypeInfoItemGroup)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class OperationTypeClassesInfo(TypeInfoItemGroup):
_kiara_model_id = "info.operation_types"
@classmethod
def base_info_class(cls) -> Type[OperationTypeInfo]: # type: ignore
return OperationTypeInfo
type_name: Literal["operation_type"] = "operation_type"
item_infos: Mapping[str, OperationTypeInfo] = Field( # type: ignore
description="The operation info instances for each type."
)
OperationTypeInfo (TypeInfo)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class OperationTypeInfo(TypeInfo[Type["OperationType"]]):
_kiara_model_id = "info.operation_type"
@classmethod
def create_from_type_class( # type: ignore
cls, kiara: "Kiara", type_cls: Type["OperationType"] # type: ignore
) -> "OperationTypeInfo":
authors_md = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
python_class = PythonClass.from_class(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
return OperationTypeInfo.construct(
**{
"type_name": type_cls._operation_type_name, # type: ignore
"documentation": doc,
"authors": authors_md,
"context": properties_md,
"python_class": python_class,
}
)
@classmethod
def base_class(self) -> Type["OperationType"]:
from kiara.operations import OperationType
return OperationType
@classmethod
def category_name(cls) -> str:
return "operation_type"
def _retrieve_id(self) -> str:
return self.type_name
def _retrieve_data_to_hash(self) -> Any:
return self.type_name
base_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_class(self) -> Type["OperationType"]:
from kiara.operations import OperationType
return OperationType
category_name()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def category_name(cls) -> str:
return "operation_type"
create_from_type_class(kiara, type_cls)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_type_class( # type: ignore
cls, kiara: "Kiara", type_cls: Type["OperationType"] # type: ignore
) -> "OperationTypeInfo":
authors_md = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
python_class = PythonClass.from_class(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
return OperationTypeInfo.construct(
**{
"type_name": type_cls._operation_type_name, # type: ignore
"documentation": doc,
"authors": authors_md,
"context": properties_md,
"python_class": python_class,
}
)
TypeInfo (ItemInfo, Generic)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class TypeInfo(ItemInfo, Generic[INFO_BASE_CLASS]):
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: INFO_BASE_CLASS, **kwargs):
return cls.create_from_type_class(type_cls=instance, kiara=kiara)
@classmethod
@abc.abstractmethod
def create_from_type_class(
self, type_cls: INFO_BASE_CLASS, kiara: "Kiara"
) -> "ItemInfo":
pass
@classmethod
def base_instance_class(self) -> INFO_BASE_CLASS:
return type # type: ignore
python_class: PythonClass = Field(
description="The python class that implements this module type."
)
python_class: PythonClass
pydantic-field
required
¶The python class that implements this module type.
base_instance_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_instance_class(self) -> INFO_BASE_CLASS:
return type # type: ignore
create_from_instance(kiara, instance, **kwargs)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: INFO_BASE_CLASS, **kwargs):
return cls.create_from_type_class(type_cls=instance, kiara=kiara)
create_from_type_class(type_cls, kiara)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
@abc.abstractmethod
def create_from_type_class(
self, type_cls: INFO_BASE_CLASS, kiara: "Kiara"
) -> "ItemInfo":
pass
TypeInfoItemGroup (InfoItemGroup)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class TypeInfoItemGroup(InfoItemGroup[TypeInfo]):
@classmethod
@abc.abstractmethod
def base_info_class(cls) -> Type[TypeInfo]:
pass
@classmethod
def create_from_type_items(
cls, kiara: "Kiara", group_title: Union[str, None] = None, **items: Type
) -> "TypeInfoItemGroup":
type_infos = {
k: cls.base_info_class().create_from_type_class(type_cls=v, kiara=kiara)
for k, v in items.items()
}
data_types_info = cls.construct(group_alias=group_title, item_infos=type_infos) # type: ignore
return data_types_info
base_info_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
@abc.abstractmethod
def base_info_class(cls) -> Type[TypeInfo]:
pass
create_from_type_items(kiara, group_title=None, **items)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_type_items(
cls, kiara: "Kiara", group_title: Union[str, None] = None, **items: Type
) -> "TypeInfoItemGroup":
type_infos = {
k: cls.base_info_class().create_from_type_class(type_cls=v, kiara=kiara)
for k, v in items.items()
}
data_types_info = cls.construct(group_alias=group_title, item_infos=type_infos) # type: ignore
return data_types_info
ValueInfo (ItemInfo)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class ValueInfo(ItemInfo[Value]):
_kiara_model_id = "info.value"
@classmethod
def base_instance_class(cls) -> Type[Value]:
return Value
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: Value, **kwargs: Any):
resolve_aliases = kwargs.get("resolve_aliases", True)
resolve_destinies = kwargs.get("resolve_destinies", True)
resolve_properties = kwargs.get("resolve_properties", True)
if resolve_aliases:
aliases = sorted(
kiara.alias_registry.find_aliases_for_value_id(instance.value_id)
)
else:
aliases = None
if instance.is_stored:
persisted_details = kiara.data_registry.retrieve_persisted_value_details(
value_id=instance.value_id
)
else:
persisted_details = None
if instance.data_type_name in kiara.type_registry.data_type_profiles:
is_internal = "internal" in kiara.type_registry.get_type_lineage(
instance.data_type_name
)
else:
is_internal = False
if resolve_destinies:
destiny_links = kiara.data_registry.find_destinies_for_value(
value_id=instance.value_id
)
filtered_destinies = {}
for alias, value_id in destiny_links.items():
if (
alias in instance.property_links.keys()
and value_id == instance.property_links[alias]
):
continue
filtered_destinies[alias] = value_id
else:
filtered_destinies = None
if resolve_properties:
properties = instance.get_all_property_data()
else:
properties = None
authors = AuthorsMetadataModel()
context = ContextMetadataModel()
doc = DocumentationMetadataModel()
model = ValueInfo(
type_name=str(instance.value_id),
documentation=doc,
authors=authors,
context=context,
value_id=instance.value_id,
kiara_id=instance.kiara_id,
value_schema=instance.value_schema,
value_status=instance.value_status,
environment_hashes=instance.environment_hashes,
value_size=instance.value_size,
value_hash=instance.value_hash,
pedigree=instance.pedigree,
pedigree_output_name=instance.pedigree_output_name,
data_type_info=instance.data_type_info,
# data_type_config=instance.data_type_config,
# data_type_class=instance.data_type_class,
property_links=instance.property_links,
destiny_links=filtered_destinies,
destiny_backlinks=instance.destiny_backlinks,
aliases=aliases,
serialized=persisted_details,
properties=properties,
is_internal=is_internal,
is_persisted=instance._is_stored,
)
model._value = instance
model._alias_registry = kiara.alias_registry # type: ignore
model._data_registry = instance._data_registry
return model
value_id: uuid.UUID = Field(description="The id of the value.")
kiara_id: uuid.UUID = Field(
description="The id of the kiara context this value belongs to."
)
value_schema: ValueSchema = Field(
description="The schema that was used for this Value."
)
value_status: ValueStatus = Field(description="The set/unset status of this value.")
value_size: int = Field(description="The size of this value, in bytes.")
value_hash: str = Field(description="The hash of this value.")
pedigree: ValuePedigree = Field(
description="Information about the module and inputs that went into creating this value."
)
pedigree_output_name: str = Field(
description="The output name that produced this value (using the manifest inside the pedigree)."
)
data_type_info: DataTypeInfo = Field(
description="Information about the underlying data type and it's configuration."
)
aliases: Union[List[str], None] = Field(
description="The aliases that are registered for this value."
)
serialized: Union[PersistedData, None] = Field(
description="Details for the serialization process that was used for this value."
)
properties: Union[Mapping[str, Any], None] = Field(
description="Property data for this value.", default=None
)
destiny_links: Union[Mapping[str, uuid.UUID], None] = Field(
description="References to all the values that act as destiny for this value in this context."
)
environment_hashes: Mapping[str, Mapping[str, str]] = Field(
description="Hashes for the environments this value was created in."
)
enviroments: Union[Mapping[str, Mapping[str, Any]], None] = Field(
description="Information about the environments this value was created in.",
default=None,
)
property_links: Mapping[str, uuid.UUID] = Field(
description="Links to values that are properties of this value.",
default_factory=dict,
)
destiny_backlinks: Mapping[uuid.UUID, str] = Field(
description="Backlinks to values that this value acts as destiny/or property for.",
default_factory=dict,
)
is_internal: bool = Field(
description="Whether this value is only used internally in kiara.",
default=False,
)
is_persisted: bool = Field(
description="Whether this value is stored in at least one data store."
)
_alias_registry: "AliasRegistry" = PrivateAttr(default=None)
_data_registry: "DataRegistry" = PrivateAttr(default=None)
_value: Value = PrivateAttr(default=None)
def _retrieve_id(self) -> str:
return str(self.value_id)
def _retrieve_data_to_hash(self) -> Any:
return self.value_id.bytes
@property
def property_values(self) -> "ValueMap":
return self._value.property_values
@property
def lineage(self) -> "ValueLineage":
return self._value.lineage
def resolve_aliases(self):
if self.aliases is None:
aliases = self._alias_registry.find_aliases_for_value_id(self.value_id)
if aliases:
aliases = sorted(aliases)
self.aliases = aliases
def resolve_destinies(self):
if self.destiny_links is None:
destiny_links = self._value._data_registry.find_destinies_for_value(
value_id=self.value_id
)
filtered_destinies = {}
for alias, value_id in destiny_links.items():
if (
alias in self.property_links.keys()
and value_id == self.property_links[alias]
):
continue
filtered_destinies[alias] = value_id
self.destiny_links = filtered_destinies
def create_info_data(self, **config: Any) -> Mapping[str, Any]:
return self._value.create_info_data(**config)
def create_renderable(self, **render_config: Any) -> RenderableType:
return self._value.create_renderable(**render_config)
aliases: List[str]
pydantic-field
¶The aliases that are registered for this value.
data_type_info: DataTypeInfo
pydantic-field
required
¶Information about the underlying data type and it's configuration.
destiny_backlinks: Mapping[uuid.UUID, str]
pydantic-field
¶Backlinks to values that this value acts as destiny/or property for.
destiny_links: Mapping[str, uuid.UUID]
pydantic-field
¶References to all the values that act as destiny for this value in this context.
enviroments: Mapping[str, Mapping[str, Any]]
pydantic-field
¶Information about the environments this value was created in.
environment_hashes: Mapping[str, Mapping[str, str]]
pydantic-field
required
¶Hashes for the environments this value was created in.
is_internal: bool
pydantic-field
¶Whether this value is only used internally in kiara.
is_persisted: bool
pydantic-field
required
¶Whether this value is stored in at least one data store.
kiara_id: UUID
pydantic-field
required
¶The id of the kiara context this value belongs to.
lineage: ValueLineage
property
readonly
¶pedigree: ValuePedigree
pydantic-field
required
¶Information about the module and inputs that went into creating this value.
pedigree_output_name: str
pydantic-field
required
¶The output name that produced this value (using the manifest inside the pedigree).
properties: Mapping[str, Any]
pydantic-field
¶Property data for this value.
property_links: Mapping[str, uuid.UUID]
pydantic-field
¶Links to values that are properties of this value.
property_values: ValueMap
property
readonly
¶serialized: PersistedData
pydantic-field
¶Details for the serialization process that was used for this value.
value_hash: str
pydantic-field
required
¶The hash of this value.
value_id: UUID
pydantic-field
required
¶The id of the value.
value_schema: ValueSchema
pydantic-field
required
¶The schema that was used for this Value.
value_size: int
pydantic-field
required
¶The size of this value, in bytes.
value_status: ValueStatus
pydantic-field
required
¶The set/unset status of this value.
base_instance_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_instance_class(cls) -> Type[Value]:
return Value
create_from_instance(kiara, instance, **kwargs)
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: Value, **kwargs: Any):
resolve_aliases = kwargs.get("resolve_aliases", True)
resolve_destinies = kwargs.get("resolve_destinies", True)
resolve_properties = kwargs.get("resolve_properties", True)
if resolve_aliases:
aliases = sorted(
kiara.alias_registry.find_aliases_for_value_id(instance.value_id)
)
else:
aliases = None
if instance.is_stored:
persisted_details = kiara.data_registry.retrieve_persisted_value_details(
value_id=instance.value_id
)
else:
persisted_details = None
if instance.data_type_name in kiara.type_registry.data_type_profiles:
is_internal = "internal" in kiara.type_registry.get_type_lineage(
instance.data_type_name
)
else:
is_internal = False
if resolve_destinies:
destiny_links = kiara.data_registry.find_destinies_for_value(
value_id=instance.value_id
)
filtered_destinies = {}
for alias, value_id in destiny_links.items():
if (
alias in instance.property_links.keys()
and value_id == instance.property_links[alias]
):
continue
filtered_destinies[alias] = value_id
else:
filtered_destinies = None
if resolve_properties:
properties = instance.get_all_property_data()
else:
properties = None
authors = AuthorsMetadataModel()
context = ContextMetadataModel()
doc = DocumentationMetadataModel()
model = ValueInfo(
type_name=str(instance.value_id),
documentation=doc,
authors=authors,
context=context,
value_id=instance.value_id,
kiara_id=instance.kiara_id,
value_schema=instance.value_schema,
value_status=instance.value_status,
environment_hashes=instance.environment_hashes,
value_size=instance.value_size,
value_hash=instance.value_hash,
pedigree=instance.pedigree,
pedigree_output_name=instance.pedigree_output_name,
data_type_info=instance.data_type_info,
# data_type_config=instance.data_type_config,
# data_type_class=instance.data_type_class,
property_links=instance.property_links,
destiny_links=filtered_destinies,
destiny_backlinks=instance.destiny_backlinks,
aliases=aliases,
serialized=persisted_details,
properties=properties,
is_internal=is_internal,
is_persisted=instance._is_stored,
)
model._value = instance
model._alias_registry = kiara.alias_registry # type: ignore
model._data_registry = instance._data_registry
return model
create_info_data(self, **config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_info_data(self, **config: Any) -> Mapping[str, Any]:
return self._value.create_info_data(**config)
create_renderable(self, **render_config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, **render_config: Any) -> RenderableType:
return self._value.create_renderable(**render_config)
resolve_aliases(self)
¶Source code in kiara/interfaces/python_api/models/info.py
def resolve_aliases(self):
if self.aliases is None:
aliases = self._alias_registry.find_aliases_for_value_id(self.value_id)
if aliases:
aliases = sorted(aliases)
self.aliases = aliases
resolve_destinies(self)
¶Source code in kiara/interfaces/python_api/models/info.py
def resolve_destinies(self):
if self.destiny_links is None:
destiny_links = self._value._data_registry.find_destinies_for_value(
value_id=self.value_id
)
filtered_destinies = {}
for alias, value_id in destiny_links.items():
if (
alias in self.property_links.keys()
and value_id == self.property_links[alias]
):
continue
filtered_destinies[alias] = value_id
self.destiny_links = filtered_destinies
ValueTypeAndDescription (BaseModel)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class ValueTypeAndDescription(BaseModel):
description: str = Field(description="The description for the value.")
type: str = Field(description="The value type.")
value_default: Any = Field(description="Default for the value.", default=None)
required: bool = Field(description="Whether this value is required")
ValuesInfo (InfoItemGroup)
pydantic-model
¶Source code in kiara/interfaces/python_api/models/info.py
class ValuesInfo(InfoItemGroup[ValueInfo]):
@classmethod
def base_info_class(cls) -> Type[ValueInfo]:
return ValueInfo
def create_render_map(
self, render_type: str, default_render_func: Callable, **render_config
):
list_by_alias = render_config.get("list_by_alias", True)
show_internal = render_config.get("show_internal_values", False)
render_fields = render_config.get("render_fields", None)
if not render_fields:
render_fields = [k for k, v in RENDER_FIELDS.items() if v["show_default"]]
if list_by_alias:
render_fields[0] = "aliases"
render_fields[1] = "value_id"
render_map: Dict[uuid.UUID, Dict[str, Any]] = {}
lookup = {}
value_info: ValueInfo
for value_info in self.item_infos.values(): # type: ignore
if not show_internal and value_info.is_internal:
continue
lookup[value_info.value_id] = value_info
details = {}
for property in render_fields:
render_func = (
RENDER_FIELDS.get(property, {})
.get("render", {})
.get(render_type, None)
)
if render_func is None:
if hasattr(value_info, property):
attr = getattr(value_info, property)
rendered = default_render_func(attr)
else:
raise Exception(
f"Can't render property '{property}': no render function registered and not a property."
)
else:
rendered = render_func(value_info)
details[property] = rendered
render_map[value_info.value_id] = details
if not list_by_alias:
return {str(k): v for k, v in render_map.items()}
else:
result: Dict[str, Dict[str, Any]] = {}
for value_id, render_details in render_map.items():
value_aliases = lookup[value_id].aliases
if value_aliases:
for alias in value_aliases:
assert alias not in result.keys()
render_details = dict(render_details)
render_details["alias"] = alias
result[alias] = render_details
else:
render_details["alias"] = ""
result[f"no_aliases_{value_id}"] = render_details
return result
def create_renderable(self, render_type: str = "terminal", **render_config: Any):
render_map = self.create_render_map(
render_type=render_type,
default_render_func=extract_renderable,
**render_config,
)
list_by_alias = render_config.get("list_by_alias", True)
render_fields = render_config.get("render_fields", None)
if not render_fields:
render_fields = [k for k, v in RENDER_FIELDS.items() if v["show_default"]]
if list_by_alias:
render_fields.insert(0, "alias")
render_fields.remove("aliases")
table = Table(box=box.SIMPLE)
for property in render_fields:
if property == "aliases" and list_by_alias:
table.add_column("alias")
elif property == "size":
table.add_column("size", justify="right")
else:
table.add_column(property)
for item_id, details in render_map.items():
row = []
for field in render_fields:
value = details[field]
row.append(value)
table.add_row(*row)
return table
base_info_class()
classmethod
¶Source code in kiara/interfaces/python_api/models/info.py
@classmethod
def base_info_class(cls) -> Type[ValueInfo]:
return ValueInfo
create_render_map(self, render_type, default_render_func, **render_config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_render_map(
self, render_type: str, default_render_func: Callable, **render_config
):
list_by_alias = render_config.get("list_by_alias", True)
show_internal = render_config.get("show_internal_values", False)
render_fields = render_config.get("render_fields", None)
if not render_fields:
render_fields = [k for k, v in RENDER_FIELDS.items() if v["show_default"]]
if list_by_alias:
render_fields[0] = "aliases"
render_fields[1] = "value_id"
render_map: Dict[uuid.UUID, Dict[str, Any]] = {}
lookup = {}
value_info: ValueInfo
for value_info in self.item_infos.values(): # type: ignore
if not show_internal and value_info.is_internal:
continue
lookup[value_info.value_id] = value_info
details = {}
for property in render_fields:
render_func = (
RENDER_FIELDS.get(property, {})
.get("render", {})
.get(render_type, None)
)
if render_func is None:
if hasattr(value_info, property):
attr = getattr(value_info, property)
rendered = default_render_func(attr)
else:
raise Exception(
f"Can't render property '{property}': no render function registered and not a property."
)
else:
rendered = render_func(value_info)
details[property] = rendered
render_map[value_info.value_id] = details
if not list_by_alias:
return {str(k): v for k, v in render_map.items()}
else:
result: Dict[str, Dict[str, Any]] = {}
for value_id, render_details in render_map.items():
value_aliases = lookup[value_id].aliases
if value_aliases:
for alias in value_aliases:
assert alias not in result.keys()
render_details = dict(render_details)
render_details["alias"] = alias
result[alias] = render_details
else:
render_details["alias"] = ""
result[f"no_aliases_{value_id}"] = render_details
return result
create_renderable(self, render_type='terminal', **render_config)
¶Source code in kiara/interfaces/python_api/models/info.py
def create_renderable(self, render_type: str = "terminal", **render_config: Any):
render_map = self.create_render_map(
render_type=render_type,
default_render_func=extract_renderable,
**render_config,
)
list_by_alias = render_config.get("list_by_alias", True)
render_fields = render_config.get("render_fields", None)
if not render_fields:
render_fields = [k for k, v in RENDER_FIELDS.items() if v["show_default"]]
if list_by_alias:
render_fields.insert(0, "alias")
render_fields.remove("aliases")
table = Table(box=box.SIMPLE)
for property in render_fields:
if property == "aliases" and list_by_alias:
table.add_column("alias")
elif property == "size":
table.add_column("size", justify="right")
else:
table.add_column(property)
for item_id, details in render_map.items():
row = []
for field in render_fields:
value = details[field]
row.append(value)
table.add_row(*row)
return table
pretty_print_value_data_terminal(value)
¶Source code in kiara/interfaces/python_api/models/info.py
def pretty_print_value_data_terminal(value: "ValueInfo"):
try:
renderable = value._value._data_registry.pretty_print_data(
value.value_id, target_type="terminal_renderable"
)
except Exception as e:
log_exception(e)
log_message("error.pretty_print", value=value.value_id, error=e)
renderable = [str(value._value.data)]
return renderable
job
¶
JobDesc (BaseModel)
pydantic-model
¶An object describing a compute job with both raw or referenced inputs.
Source code in kiara/interfaces/python_api/models/job.py
class JobDesc(BaseModel):
"""An object describing a compute job with both raw or referenced inputs."""
module_type: str = Field(description="The module type.")
module_config: Dict[str, Any] = Field(
default_factory=dict, description="The configuration for the module."
)
inputs: Dict[str, Any] = Field(description="The inputs for the job.")
def create_job_config(self, kiara: Kiara) -> JobConfig:
manifest = Manifest(
module_type=self.module_type, module_config=self.module_config
)
module = kiara.create_module(manifest=manifest)
job_config = JobConfig.create_from_module(
data_registry=kiara.data_registry, module=module, inputs=self.inputs
)
return job_config
inputs: Dict[str, Any]
pydantic-field
required
¶The inputs for the job.
module_config: Dict[str, Any]
pydantic-field
¶The configuration for the module.
module_type: str
pydantic-field
required
¶The module type.
create_job_config(self, kiara)
¶Source code in kiara/interfaces/python_api/models/job.py
def create_job_config(self, kiara: Kiara) -> JobConfig:
manifest = Manifest(
module_type=self.module_type, module_config=self.module_config
)
module = kiara.create_module(manifest=manifest)
job_config = JobConfig.create_from_module(
data_registry=kiara.data_registry, module=module, inputs=self.inputs
)
return job_config
operation
¶
KiaraOperation
¶A class to provide a convenience API around executing a specific operation.
Source code in kiara/interfaces/python_api/operation.py
class KiaraOperation(object):
"""A class to provide a convenience API around executing a specific operation."""
def __init__(
self,
kiara: "Kiara",
operation_name: str,
operation_config: Union[Mapping[str, Any], None] = None,
):
self._kiara: Kiara = kiara
self._operation_name: str = operation_name
if operation_config is None:
operation_config = {}
else:
operation_config = dict(operation_config)
self._operation_config: Dict[str, Any] = operation_config
self._inputs_raw: Dict[str, Any] = {}
self._operation: Union[Operation, None] = None
self._inputs: Union[ValueMap, None] = None
self._job_config: Union[JobConfig, None] = None
self._queued_jobs: Dict[uuid.UUID, Dict[str, Any]] = {}
self._last_job: Union[uuid.UUID, None] = None
self._results: Dict[uuid.UUID, ValueMap] = {}
self._defaults: Union[Dict[str, Any], None] = None
def validate(self):
self.job_config # noqa
def _invalidate(self):
self._job_config = None
# self._defaults = None
@property
def operation_inputs(self) -> ValueMap:
if self._inputs is not None:
return self._inputs
self._invalidate()
if self._defaults is not None:
data = dict(self._defaults)
else:
data = {}
data.update(self._inputs_raw)
self._inputs = self._kiara.data_registry.create_valuemap(
data, self.operation.inputs_schema
)
return self._inputs
def set_input(self, field: Union[str, None], value: Any = None):
if field is None:
if value is None:
self._inputs_raw.clear()
self._invalidate()
return
else:
if not isinstance(value, Mapping):
raise Exception(
"Can't set inputs dictionary (if no key is provided, value must be 'None' or of type 'Mapping')."
)
self._inputs_raw.clear()
self.set_inputs(**value)
self._invalidate()
return
else:
old = self._inputs_raw.get(field, None)
self._inputs_raw[field] = value
if old != value:
self._invalidate()
return
def set_inputs(self, **inputs: Any):
changed = False
for k, v in inputs.items():
old = self._inputs_raw.get(k, None)
self._inputs_raw[k] = v
if old != v:
changed = True
if changed:
self._invalidate()
return
def run(self, **inputs: Any) -> ValueMap:
job_id = self.queue_job(**inputs)
results = self.retrieve_result(job_id=job_id)
return results
@property
def operation_name(self) -> str:
return self._operation_name
@operation_name.setter
def operation_name(self, operation_name: str):
self._operation_name = operation_name
self._operation = None
@property
def operation_config(self) -> Mapping[str, Any]:
return self._operation_config
def set_operation_config_value(
self, key: Union[str, None], value: Any = None
) -> Mapping[str, Any]:
if key is None:
if value is None:
old = bool(self._operation_config)
self._operation_config.clear()
if old:
self._operation = None
return self._operation_config
else:
try:
old_conf = self._operation_config
self._operation_config = dict(value)
if old_conf != self._operation_config:
self._operation = None
return self._operation_config
except Exception as e:
raise Exception(
f"Can't set configuration value dictionary (if no key is provided, value must be 'None' or of type 'Mapping'): {e}"
)
self._operation_config[key] = value
self._invalidate()
return self._operation_config
@property
def operation(self) -> "Operation":
if self._operation is not None:
return self._operation
self._invalidate()
self._defaults = None
operation = create_operation(
module_or_operation=self._operation_name,
operation_config=self.operation_config,
kiara=self._kiara,
)
if os.path.isfile(self._operation_name):
data = get_data_from_file(self._operation_name)
self._defaults = data.get("inputs", {})
self._operation = operation
return self._operation
@property
def job_config(self) -> JobConfig:
if self._job_config is not None:
return self._job_config
self._job_config = self.operation.prepare_job_config(
kiara=self._kiara, inputs=self.operation_inputs
)
return self._job_config
def queue_job(self, **inputs) -> uuid.UUID:
if inputs:
self.set_inputs(**inputs)
job_config = self.job_config
operation = self.operation
op_inputs = self.operation_inputs
job_id = self._kiara.job_registry.execute_job(job_config=job_config, wait=False)
self._queued_jobs[job_id] = {
"job_config": job_config,
"operation": operation,
"inputs": op_inputs,
}
self._last_job = job_id
return job_id
def retrieve_result(self, job_id: Union[uuid.UUID, None] = None) -> ValueMap:
if job_id in self._results.keys():
assert job_id is not None
return self._results[job_id]
if job_id is None:
job_id = self._last_job
if job_id is None:
raise Exception("No job queued (yet).")
operation: Operation = self._queued_jobs[job_id]["operation"] # type: ignore
status = self._kiara.job_registry.get_job_status(job_id=job_id)
if status == JobStatus.FAILED:
job = self._kiara.job_registry.get_active_job(job_id=job_id)
raise FailedJobException(job=job)
outputs = self._kiara.job_registry.retrieve_result(job_id)
outputs = operation.process_job_outputs(outputs=outputs)
self._results[job_id] = outputs
return outputs
def save_result(
self,
job_id: Union[uuid.UUID, None] = None,
aliases: Union[None, str, Mapping] = None,
) -> StoreValuesResult:
if job_id is None:
job_id = self._last_job
if job_id is None:
raise Exception("No job queued (yet).")
result = self.retrieve_result(job_id=job_id)
alias_map = create_save_config(field_names=result.field_names, aliases=aliases)
store_result = self._kiara.save_values(values=result, alias_map=alias_map)
# if self.operation.module.characteristics.is_idempotent:
self._kiara.job_registry.store_job_record(job_id=job_id)
return store_result
def create_renderable(self, **config: Any) -> RenderableType:
show_operation_name = config.get("show_operation_name", True)
show_operation_doc = config.get("show_operation_doc", True)
show_inputs = config.get("show_inputs", False)
show_outputs_schema = config.get("show_outputs_schema", False)
show_headers = config.get("show_headers", True)
items: List[Any] = []
if show_operation_name:
items.append(f"Operation: [bold]{self.operation_name}[/bold]")
if show_operation_doc and self.operation.doc.is_set:
items.append("")
items.append(Markdown(self.operation.doc.full_doc, style="i"))
if show_inputs:
if show_headers:
items.append("\nInputs:")
try:
op_inputs = self.operation_inputs
inputs: Any = create_value_map_status_renderable(op_inputs)
except InvalidValuesException as ive:
inputs = ive.create_renderable(**config)
except Exception as e:
inputs = f"[red bold]{e}[/red bold]"
items.append(inputs)
if show_outputs_schema:
if show_headers:
items.append("\nOutputs:")
outputs_schema = create_table_from_field_schemas(
_add_default=False,
_add_required=False,
_show_header=True,
_constants=None,
fields=self.operation.outputs_schema,
)
items.append(outputs_schema)
return Group(*items)
job_config: JobConfig
property
readonly
¶operation: Operation
property
readonly
¶operation_config: Mapping[str, Any]
property
readonly
¶operation_inputs: ValueMap
property
readonly
¶operation_name: str
property
writable
¶create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/operation.py
def create_renderable(self, **config: Any) -> RenderableType:
show_operation_name = config.get("show_operation_name", True)
show_operation_doc = config.get("show_operation_doc", True)
show_inputs = config.get("show_inputs", False)
show_outputs_schema = config.get("show_outputs_schema", False)
show_headers = config.get("show_headers", True)
items: List[Any] = []
if show_operation_name:
items.append(f"Operation: [bold]{self.operation_name}[/bold]")
if show_operation_doc and self.operation.doc.is_set:
items.append("")
items.append(Markdown(self.operation.doc.full_doc, style="i"))
if show_inputs:
if show_headers:
items.append("\nInputs:")
try:
op_inputs = self.operation_inputs
inputs: Any = create_value_map_status_renderable(op_inputs)
except InvalidValuesException as ive:
inputs = ive.create_renderable(**config)
except Exception as e:
inputs = f"[red bold]{e}[/red bold]"
items.append(inputs)
if show_outputs_schema:
if show_headers:
items.append("\nOutputs:")
outputs_schema = create_table_from_field_schemas(
_add_default=False,
_add_required=False,
_show_header=True,
_constants=None,
fields=self.operation.outputs_schema,
)
items.append(outputs_schema)
return Group(*items)
queue_job(self, **inputs)
¶Source code in kiara/interfaces/python_api/operation.py
def queue_job(self, **inputs) -> uuid.UUID:
if inputs:
self.set_inputs(**inputs)
job_config = self.job_config
operation = self.operation
op_inputs = self.operation_inputs
job_id = self._kiara.job_registry.execute_job(job_config=job_config, wait=False)
self._queued_jobs[job_id] = {
"job_config": job_config,
"operation": operation,
"inputs": op_inputs,
}
self._last_job = job_id
return job_id
retrieve_result(self, job_id=None)
¶Source code in kiara/interfaces/python_api/operation.py
def retrieve_result(self, job_id: Union[uuid.UUID, None] = None) -> ValueMap:
if job_id in self._results.keys():
assert job_id is not None
return self._results[job_id]
if job_id is None:
job_id = self._last_job
if job_id is None:
raise Exception("No job queued (yet).")
operation: Operation = self._queued_jobs[job_id]["operation"] # type: ignore
status = self._kiara.job_registry.get_job_status(job_id=job_id)
if status == JobStatus.FAILED:
job = self._kiara.job_registry.get_active_job(job_id=job_id)
raise FailedJobException(job=job)
outputs = self._kiara.job_registry.retrieve_result(job_id)
outputs = operation.process_job_outputs(outputs=outputs)
self._results[job_id] = outputs
return outputs
run(self, **inputs)
¶Source code in kiara/interfaces/python_api/operation.py
def run(self, **inputs: Any) -> ValueMap:
job_id = self.queue_job(**inputs)
results = self.retrieve_result(job_id=job_id)
return results
save_result(self, job_id=None, aliases=None)
¶Source code in kiara/interfaces/python_api/operation.py
def save_result(
self,
job_id: Union[uuid.UUID, None] = None,
aliases: Union[None, str, Mapping] = None,
) -> StoreValuesResult:
if job_id is None:
job_id = self._last_job
if job_id is None:
raise Exception("No job queued (yet).")
result = self.retrieve_result(job_id=job_id)
alias_map = create_save_config(field_names=result.field_names, aliases=aliases)
store_result = self._kiara.save_values(values=result, alias_map=alias_map)
# if self.operation.module.characteristics.is_idempotent:
self._kiara.job_registry.store_job_record(job_id=job_id)
return store_result
set_input(self, field, value=None)
¶Source code in kiara/interfaces/python_api/operation.py
def set_input(self, field: Union[str, None], value: Any = None):
if field is None:
if value is None:
self._inputs_raw.clear()
self._invalidate()
return
else:
if not isinstance(value, Mapping):
raise Exception(
"Can't set inputs dictionary (if no key is provided, value must be 'None' or of type 'Mapping')."
)
self._inputs_raw.clear()
self.set_inputs(**value)
self._invalidate()
return
else:
old = self._inputs_raw.get(field, None)
self._inputs_raw[field] = value
if old != value:
self._invalidate()
return
set_inputs(self, **inputs)
¶Source code in kiara/interfaces/python_api/operation.py
def set_inputs(self, **inputs: Any):
changed = False
for k, v in inputs.items():
old = self._inputs_raw.get(k, None)
self._inputs_raw[k] = v
if old != v:
changed = True
if changed:
self._invalidate()
return
set_operation_config_value(self, key, value=None)
¶Source code in kiara/interfaces/python_api/operation.py
def set_operation_config_value(
self, key: Union[str, None], value: Any = None
) -> Mapping[str, Any]:
if key is None:
if value is None:
old = bool(self._operation_config)
self._operation_config.clear()
if old:
self._operation = None
return self._operation_config
else:
try:
old_conf = self._operation_config
self._operation_config = dict(value)
if old_conf != self._operation_config:
self._operation = None
return self._operation_config
except Exception as e:
raise Exception(
f"Can't set configuration value dictionary (if no key is provided, value must be 'None' or of type 'Mapping'): {e}"
)
self._operation_config[key] = value
self._invalidate()
return self._operation_config
validate(self)
¶Source code in kiara/interfaces/python_api/operation.py
def validate(self):
self.job_config # noqa
utils
¶create_save_config(field_names, aliases)
¶Source code in kiara/interfaces/python_api/utils.py
def create_save_config(
field_names: Union[str, Iterable[str]],
aliases: Union[None, str, Iterable[str], Mapping[str, Any]],
) -> Dict[str, List[str]]:
if isinstance(field_names, str):
field_names = [field_names]
if aliases is None:
alias_map: Dict[str, List[str]] = {}
elif isinstance(aliases, str):
alias_map = {}
for field_name in field_names:
alias_map[field_name] = [f"{aliases}.{field_name}"]
elif isinstance(aliases, Mapping):
alias_map = {}
for field_name in aliases.keys():
if field_name in field_names:
if isinstance(aliases[field_name], str):
alias_map[field_name] = [aliases[field_name]]
else:
alias_map[field_name] = sorted(aliases[field_name])
else:
logger.warning(
"ignore.field_alias",
ignored_field_name=field_name,
reason="field name not in results",
available_field_names=sorted(field_names),
)
continue
else:
raise Exception(
f"Invalid type '{type(aliases)}' for aliases parameter, must be string or mapping."
)
return alias_map
value
¶logger
¶
StoreValueResult (BaseModel)
pydantic-model
¶Source code in kiara/interfaces/python_api/value.py
class StoreValueResult(BaseModel):
value: Value = Field(description="The stored value.")
aliases: List[str] = Field(
description="The aliases that where assigned to the value when stored."
)
persisted_data: Union[None, PersistedData] = Field(
description="The structure describing the data that was persisted, 'None' if the data was already stored before (or storing failed)."
)
error: Union[str, None] = Field(
description="An error that occured while trying to store."
)
aliases: List[str]
pydantic-field
required
¶The aliases that where assigned to the value when stored.
error: str
pydantic-field
¶An error that occured while trying to store.
persisted_data: PersistedData
pydantic-field
¶The structure describing the data that was persisted, 'None' if the data was already stored before (or storing failed).
value: Value
pydantic-field
required
¶The stored value.
StoreValuesResult (BaseModel)
pydantic-model
¶Source code in kiara/interfaces/python_api/value.py
class StoreValuesResult(BaseModel):
__root__: Dict[str, StoreValueResult]
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=True, show_lines=False, box=box.SIMPLE)
table.add_column("field", style="b")
table.add_column("data type", style="i")
table.add_column("stored id", style="i")
table.add_column("alias(es)")
for field_name, value_result in self.__root__.items():
row = [
field_name,
str(value_result.value.value_schema.type),
str(value_result.value.value_id),
]
if value_result.aliases:
row.append(", ".join(value_result.aliases))
else:
row.append("")
table.add_row(*row)
return table
def __len__(self):
return len(self.__root__)
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/value.py
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=True, show_lines=False, box=box.SIMPLE)
table.add_column("field", style="b")
table.add_column("data type", style="i")
table.add_column("stored id", style="i")
table.add_column("alias(es)")
for field_name, value_result in self.__root__.items():
row = [
field_name,
str(value_result.value.value_schema.type),
str(value_result.value.value_id),
]
if value_result.aliases:
row.append(", ".join(value_result.aliases))
else:
row.append("")
table.add_row(*row)
return table
workflow
¶logger
¶
Workflow
¶Source code in kiara/interfaces/python_api/workflow.py
class Workflow(object):
@classmethod
def load(cls, workflow: Union[uuid.UUID, str]):
pass
@classmethod
def create(
cls,
alias: Union[None, str] = None,
blueprint: Union[str, None] = None,
doc: Union[None, str, DocumentationMetadataModel] = None,
kiara: Union[None, "Kiara"] = None,
overwrite_existing: bool = False,
):
if kiara is None:
from kiara.context import Kiara
kiara = Kiara.instance()
operation: Union[None, KiaraOperation] = None
if blueprint:
operation = KiaraOperation(kiara=kiara, operation_name=blueprint)
if doc is None:
doc = operation.operation.doc
details = WorkflowDetails(documentation=doc)
workflow_obj = Workflow(kiara=kiara, workflow=details)
if alias:
workflow_obj.workflow_alias = alias
if blueprint:
assert operation
module = operation.operation.module
if isinstance(module.config, PipelineConfig):
config: PipelineConfig = module.config
else:
raise NotImplementedError()
workflow_obj.add_steps(*config.steps)
return workflow_obj
def __init__(
self, kiara: "Kiara", workflow: Union[uuid.UUID, WorkflowDetails, str, None]
):
self._kiara: Kiara = kiara
self._execution_context: ExecutionContext = ExecutionContext()
self._pipeline_controller: WorkflowPipelineController = (
WorkflowPipelineController(kiara=self._kiara)
)
workflow_id = None
self._workflow_alias: Union[None, str] = None
self._is_stored: bool = False
self._workflow_details: WorkflowDetails = None # type: ignore
if workflow is None:
workflow_id = None
if isinstance(workflow, WorkflowDetails):
workflow_id = workflow.workflow_id
elif isinstance(workflow, uuid.UUID):
workflow_id = workflow
elif isinstance(workflow, str):
try:
workflow_id = uuid.UUID(workflow)
except Exception:
# means it is meant to be an alias
self._workflow_alias = workflow
try:
self._workflow_details = (
self._kiara.workflow_registry.get_workflow_details(workflow)
)
self._is_stored = True
workflow_id = self._workflow_details.workflow_id
except Exception:
pass
if workflow_id:
if self._workflow_details is None:
try:
self._workflow_details = (
self._kiara.workflow_registry.get_workflow_details(
workflow=workflow_id
)
)
self._is_stored = True
except Exception:
pass
if self._workflow_details is None:
self._workflow_details = WorkflowDetails()
self._all_inputs: Dict[str, Any] = {}
self._current_inputs: Union[Dict[str, uuid.UUID], None] = None
self._current_outputs: Union[Dict[str, uuid.UUID], None] = None
self._steps: Dict[str, PipelineStep] = {}
self._pipeline_input_aliases: Dict[str, str] = {}
self._pipeline_output_aliasess: Dict[str, str] = {}
# self._current_state_cid: Union[None, CID] = None
self._state_cache: Dict[str, WorkflowState] = {}
self._job_id_cache: Dict[uuid.UUID, uuid.UUID] = {}
"""Cache to save job ids per output value(s), in order to save jobs if output values are saved."""
self._pipeline: Union[Pipeline, None] = None
self._pipeline_info: Union[PipelineInfo, None] = None
self._current_info: Union[WorkflowInfo, None] = None
self._current_state: Union[WorkflowState, None] = None
if self._workflow_details.workflow_states:
self.load_state()
@property
def workflow_id(self) -> uuid.UUID:
return self._workflow_details.workflow_id
@property
def workflow_alias(self) -> Union[None, str]:
return self._workflow_alias
@workflow_alias.setter
def workflow_alias(self, alias: str):
self._workflow_alias = alias
# TODO: register in registry
@property
def details(self) -> WorkflowDetails:
return self._workflow_details
@property
def current_inputs_schema(self) -> Mapping[str, ValueSchema]:
return self.pipeline.structure.pipeline_inputs_schema
@property
def current_input_names(self) -> List[str]:
return sorted(self.current_inputs_schema.keys())
@property
def current_outputs_schema(self) -> Mapping[str, ValueSchema]:
return self.pipeline.structure.pipeline_outputs_schema
@property
def current_output_names(self) -> List[str]:
return sorted(self.current_outputs_schema.keys())
@property
def current_inputs(self) -> Mapping[str, uuid.UUID]:
if self._current_inputs is None:
self._apply_inputs()
assert self._current_inputs is not None
return self._current_inputs
@property
def current_outputs(self) -> Mapping[str, uuid.UUID]:
if self._current_outputs is None:
try:
self.process_steps()
except Exception:
self._current_outputs = self.pipeline.get_current_pipeline_outputs()
assert self._current_outputs is not None
return self._current_outputs
@property
def current_output_values(self) -> ValueMap:
return self._kiara.data_registry.load_values(values=self.current_outputs)
@property
def current_state(self) -> WorkflowState:
if self._current_state is not None:
return self._current_state
self._current_state = WorkflowState.create_from_workflow(self)
self._state_cache[self._current_state.instance_id] = self._current_state
return self._current_state
@property
def pipeline(self) -> Pipeline:
if self._pipeline is not None:
return self._pipeline
self._invalidate_pipeline()
# if not self._steps:
# raise Exception("Can't assemble pipeline, no steps set (yet).")
steps = list(self._steps.values())
input_aliases_temp = create_input_alias_map(steps=steps)
input_aliases = {}
for k, v in input_aliases_temp.items():
if k in self._pipeline_input_aliases.keys():
input_aliases[k] = self._pipeline_input_aliases[k]
else:
input_aliases[k] = v
if not self._pipeline_output_aliasess:
output_aliases = create_output_alias_map(steps=steps)
else:
output_aliases = self._pipeline_output_aliasess
pipeline_config = PipelineConfig.from_config(
pipeline_name="__workflow__",
data={
"steps": steps,
"doc": self.details.documentation,
"input_aliases": input_aliases,
"output_aliases": output_aliases,
},
)
structure = pipeline_config.structure
self._pipeline = Pipeline(structure=structure, kiara=self._kiara)
self._pipeline_controller.pipeline = self._pipeline
return self._pipeline
def _apply_inputs(self) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
pipeline = self.pipeline
inputs_to_set = {}
for field_name, value in self._all_inputs.items():
if value in [NONE_VALUE_ID, NOT_SET_VALUE_ID]:
continue
if field_name in pipeline.structure.pipeline_inputs_schema.keys():
inputs_to_set[field_name] = value
logger.debug(
"workflow.apply_inputs",
workflow_id=str(self.workflow_id),
keys=", ".join(inputs_to_set.keys()),
)
changed: Mapping[
str, Mapping[str, Mapping[str, ChangedValue]]
] = pipeline.set_pipeline_inputs(inputs=inputs_to_set)
self._current_inputs = pipeline.get_current_pipeline_inputs()
for field_name, value_id in self._current_inputs.items():
self._all_inputs[field_name] = value_id
self._current_outputs = None
for stage, steps in pipeline.get_steps_by_stage().items():
stage_valid = True
cached_steps = []
for step_id in steps.keys():
step_details = pipeline.get_step_details(step_id=step_id)
if step_details.status == StepStatus.INPUTS_INVALID:
stage_valid = False
break
elif step_details.status == StepStatus.INPUTS_READY:
job_config = JobConfig(
module_type=step_details.step.module_type,
module_config=step_details.step.module.config.dict(),
inputs=step_details.inputs,
)
match = self._kiara.job_registry.find_matching_job_record(
inputs_manifest=job_config
)
if match:
cached_steps.append(step_id)
if cached_steps:
self.process_steps(*cached_steps)
if not stage_valid:
break
self._current_state = None
self._current_info = None
self._pipeline_info = None
return changed
def process_steps(self, *step_ids: str):
self.pipeline # noqa
if not step_ids:
output_job_map = self._pipeline_controller.process_pipeline()
else:
job_ids = {}
for step_id in step_ids:
job_id = self._pipeline_controller.process_step(
step_id=step_id, wait=True
)
job_ids[step_id] = job_id
output_job_map = self._pipeline_controller.set_processing_results(
job_ids=job_ids
)
self._job_id_cache.update(output_job_map)
self._current_outputs = self.pipeline.get_current_pipeline_outputs()
self._current_state = None
self._pipeline_info = None
self._current_info = None
def _invalidate_pipeline(self):
self._pipeline_controller.pipeline = None
self._pipeline = None
self._pipeline_info = None
self._current_info = None
self._current_state = None
def set_input(self, field_name: str, value: Any):
self.set_inputs(**{field_name: value})
def set_inputs(self, **inputs: Any):
invalid = []
for k, v in inputs.items():
if k not in self.pipeline.structure.pipeline_inputs_schema.keys():
invalid.append(k)
if invalid:
raise Exception(
f"Can't set pipeline inputs, invalid field(s): '{', '.join(invalid)}'. Available inputs: '{', '.join(self.pipeline.structure.pipeline_inputs_schema.keys())}'"
)
changed = False
for k, v in inputs.items():
# TODO: better equality test?
if k == self._all_inputs.get(k, None):
continue
self._all_inputs[k] = v
changed = True
if changed:
self._current_info = None
self._current_state = None
self._current_inputs = None
self._current_outputs = None
self._pipeline_info = None
self._apply_inputs()
def add_steps(
self,
*pipeline_steps: PipelineStep,
replace_existing: bool = False,
clear_existing: bool = False,
):
if clear_existing:
self.clear_steps()
for step in pipeline_steps:
if step.step_id in self._steps.keys() and not replace_existing:
raise Exception(
f"Can't add step with id '{step.step_id}': step with that id already exists and 'replace_existing' not set."
)
for step in pipeline_steps:
self._steps[step.step_id] = step
self._invalidate_pipeline()
def clear_steps(self, *step_ids: str):
if not step_ids:
self._steps.clear()
else:
for step_id in step_ids:
self._steps.pop(step_id, None)
self._invalidate_pipeline()
def set_input_alias(self, input_field: str, alias: str):
self._pipeline_input_aliases[input_field] = alias
self._invalidate_pipeline()
def set_output_alias(self, output_field: str, alias: str):
self._pipeline_output_aliasess[output_field] = alias
def add_step(
self,
operation: str,
step_id: Union[str, None] = None,
module_config: Union[None, Mapping[str, Any]] = None,
input_connections: Union[None, Mapping[str, str]] = None,
doc: Union[str, DocumentationMetadataModel, None] = None,
replace_existing: bool = False,
) -> PipelineStep:
"""Add a step to the workflows current pipeline structure.
If no 'step_id' is provided, a unque one will automatically be generated based on the 'module_type' argument.
Arguments:
operation: the module or operation name
step_id: the id of the new step
module_config: (optional) configuration for the kiara module this step uses
input_connections: a map with this steps input field name(s) as keys and output field links (format: <step_id>.<output_field_name>) as value(s).
replace_existing: if set to 'True', this replaces a step with the same id that already exists, otherwise an exception will be thrown
"""
if step_id is None:
step_id = find_free_id(
slugify(operation, separator="_"), current_ids=self._steps.keys()
)
if "." in step_id:
raise Exception(f"Invalid step id '{step_id}': id can't contain '.'.")
if step_id in self._steps.keys() and not replace_existing:
raise Exception(
f"Can't add step with id '{step_id}': step already exists and 'replace_existing' not set."
)
elif step_id in self._steps.keys():
raise NotImplementedError()
manifest = self._kiara.create_manifest(
module_or_operation=operation, config=module_config
)
module = self._kiara.create_module(manifest=manifest)
step = PipelineStep(
step_id=step_id,
module_type=module.module_type_name,
module_config=module.config.dict(),
module_details=KiaraModuleInstance.from_module(module=module),
doc=doc,
)
step._module = module
self._steps[step_id] = step
if input_connections:
for k, v in input_connections.items():
self.connect_to_inputs(v, f"{step_id}.{k}")
self._invalidate_pipeline()
return step
def connect_fields(self, *fields: Union[Tuple[str, str], str]):
pairs = []
current_pair = None
for field in fields:
if isinstance(field, str):
tokens = field.split(".")
if not len(tokens) == 2:
raise Exception(
f"Can't connect field '{field}', field name must be in format: <step_id>.<field_name>."
)
if not current_pair:
current_pair = [tokens]
else:
if not len(current_pair) == 1:
raise Exception(
f"Can't connect fields, invalid input(s): {fields}"
)
current_pair.append(tokens)
pairs.append(current_pair)
current_pair = None
else:
if not len(field) == 2:
raise Exception(
f"Can't connect fields, field tuples must have length 2: {field}"
)
if current_pair:
raise Exception(
f"Can't connect fields, dangling single field: {current_pair}"
)
pair = []
for f in field:
tokens = f.split(".")
if not len(tokens) == 2:
raise Exception(
f"Can't connect field '{f}', field name must be in format: <step_id>.<field_name>."
)
pair.append(tokens)
pairs.append(pair)
for pair in pairs:
self.connect_steps(pair[0][0], pair[0][1], pair[1][0], pair[1][1])
def connect_steps(
self,
source_step: Union[PipelineStep, str],
source_field: str,
target_step: Union[PipelineStep, str],
target_field: str,
):
if isinstance(source_step, str):
source_step_obj = self.get_step(source_step)
else:
source_step_obj = source_step
if isinstance(target_step, str):
target_step_obj = self.get_step(target_step)
else:
target_step_obj = target_step
source_step_id = source_step_obj.step_id
target_step_id = target_step_obj.step_id
reversed = False
if source_field not in source_step_obj.module.outputs_schema.keys():
reversed = True
if target_field not in target_step_obj.module.inputs_schema.keys():
reversed = True
if reversed:
if target_field not in target_step_obj.module.outputs_schema.keys():
raise Exception(
f"Can't connect steps '{source_step_id}.{source_field}' -> '{target_step_id}.{target_field}': invalid field name(s)."
)
if source_field not in source_step_obj.module.inputs_schema.keys():
raise Exception(
f"Can't connect steps '{source_step_id}.{source_field}' -> '{target_step_id}.{target_field}': invalid field name(s)."
)
else:
if target_field not in target_step_obj.module.inputs_schema.keys():
raise Exception(
f"Can't connect steps '{source_step_id}.{source_field}' -> '{target_step_id}.{target_field}': invalid field name(s)."
)
if source_field not in source_step_obj.module.outputs_schema.keys():
raise Exception(
f"Can't connect steps '{source_step_id}.{source_field}' -> '{target_step_id}.{target_field}': invalid field name(s)."
)
# we rely on the value of input links to always be a dict here
if not reversed:
source_addr = StepValueAddress(
step_id=source_step_id, value_name=source_field
)
target_step_obj.input_links.setdefault(target_field, []).append(source_addr) # type: ignore
else:
source_addr = StepValueAddress(
step_id=target_step_id, value_name=target_field
)
source_step_obj.input_links.setdefault(source_field, []).append(source_addr) # type: ignore
self._invalidate_pipeline()
def connect_to_inputs(self, source_field: str, *input_fields: str):
source_tokens = source_field.split(".")
if len(source_tokens) != 2:
raise Exception(
f"Can't add input link(s): invalid format for provided source '{source_field}', must be string with a single '.' to delimit step-id and output field name."
)
source_step = self.get_step(source_tokens[0])
if source_step is None:
raise Exception(
f"Can't add input link(s)': no source step with id '{source_tokens[0]}' exists."
)
if source_tokens[1] not in source_step.module.outputs_schema.keys():
av_fields = ", ".join(source_step.module.outputs_schema.keys())
raise Exception(
f"Can't add input link(s): source step with id '{source_step.step_id}' does not have output field '{source_tokens[1]}'. Available field names: {av_fields}."
)
source_addr = StepValueAddress(
step_id=source_step.step_id, value_name=source_tokens[1]
)
steps = []
for input_field in input_fields:
input_tokens = input_field.split(".")
if len(input_tokens) != 2:
raise Exception(
f"Can't add input link '{input_field}': invalid format, must be string with a single '.' to delimit step-id and field name."
)
step = self.get_step(input_tokens[0])
if step is None:
raise Exception(
f"Can't add input link '{input_field}': no step with id '{input_tokens[0]}' exists."
)
if input_tokens[1] not in step.module.inputs_schema.keys():
av_fields = ", ".join(step.module.inputs_schema.keys())
raise Exception(
f"Can't add input link '{input_field}': step with id '{input_tokens[0]}' does not have input field '{input_tokens[1]}'. Available field names: {av_fields}."
)
steps.append((step, input_tokens[1]))
for s in steps:
step, field_name = s
# we rely on the value of input links to always be a dict here
step.input_links.setdefault(field_name, []).append(source_addr) # type: ignore
self._invalidate_pipeline()
def get_step(self, step_id: str) -> PipelineStep:
step = self._steps.get(step_id, None)
if step is None:
if self._steps:
msg = f"Available step ids: {', '.join(self._steps.keys())}"
else:
msg = "Workflow does not have any steps (yet)."
raise Exception(f"No step with id '{step_id}' registered. {msg}")
return step
def load_state(
self, workflow_state_id: Union[str, None] = None
) -> Union[None, WorkflowState]:
"""Load a past state.
If no state id is specified, the latest one that was saved will be used.
Returns:
'None' if no state was loaded, otherwise the relevant 'WorkflowState' instance
"""
if workflow_state_id is None:
if not self._workflow_details.workflow_states:
return None
else:
workflow_state_id = self._workflow_details.last_state_id
if workflow_state_id is None:
raise Exception(
f"Can't load current state for workflow '{self.workflow_id}': no state available."
)
state = self._state_cache.get(workflow_state_id, None)
if state is not None:
return state
state = self._kiara.workflow_registry.get_workflow_state(
workflow=self.workflow_id, workflow_state_id=workflow_state_id
)
assert workflow_state_id == state.instance_id
self._state_cache[workflow_state_id] = state
self._all_inputs.clear()
self._current_inputs = None
self.clear_steps()
self._invalidate_pipeline()
self.add_steps(*state.steps)
self._pipeline_input_aliases = dict(state.pipeline_config.input_aliases)
self._pipeline_output_aliasess = dict(state.pipeline_config.output_aliases)
self.set_inputs(**state.inputs)
assert self._current_inputs == state.inputs
self._current_outputs = state.pipeline_info.pipeline_details.pipeline_outputs
self._pipeline_info = state.pipeline_info
self._current_state = state
self._current_info = None
return state
@property
def all_states(self) -> Mapping[str, WorkflowState]:
missing = []
for state_id in self.details.workflow_states.values():
if state_id not in self._state_cache.keys():
missing.append(state_id)
if missing:
# TODO: only request missing ones?
all_states = self._kiara.workflow_registry.get_all_states_for_workflow(
workflow=self.workflow_id
)
self._state_cache.update(all_states)
return self._state_cache
@property
def info(self) -> WorkflowInfo:
if self._current_info is not None:
return self._current_info
self._current_info = WorkflowInfo.create_from_workflow(workflow=self)
return self._current_info
@property
def pipeline_info(self) -> PipelineInfo:
if self._pipeline_info is not None:
return self._pipeline_info
self._pipeline_info = PipelineInfo.create_from_pipeline(
kiara=self._kiara, pipeline=self.pipeline
)
return self._pipeline_info
def snapshot(self, save: bool = True) -> WorkflowState:
state = self.current_state
if not self._is_stored:
aliases = []
if self._workflow_alias:
aliases.append(self._workflow_alias)
self._kiara.workflow_registry.register_workflow(
workflow_details=self._workflow_details, workflow_aliases=aliases
)
self._is_stored = True
if save:
for value in state.inputs.values():
self._kiara.data_registry.store_value(value=value)
for value in self.current_outputs.values():
if value in [NOT_SET_VALUE_ID, NONE_VALUE_ID]:
continue
self._kiara.data_registry.store_value(value=value)
job_id = self._job_id_cache[value]
try:
self._kiara.job_registry.store_job_record(job_id=job_id)
except Exception as e:
print(e)
self._workflow_details = self._kiara.workflow_registry.add_workflow_state(
workflow=self._workflow_details, workflow_state=state
)
return state
def create_renderable(self, **config: Any):
if not self._steps:
return "Invalid workflow: no steps set yet."
return self.info.create_renderable(**config)
all_states: Mapping[str, kiara.models.workflow.WorkflowState]
property
readonly
¶current_input_names: List[str]
property
readonly
¶current_inputs: Mapping[str, uuid.UUID]
property
readonly
¶current_inputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶current_output_names: List[str]
property
readonly
¶current_output_values: ValueMap
property
readonly
¶current_outputs: Mapping[str, uuid.UUID]
property
readonly
¶current_outputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶current_state: WorkflowState
property
readonly
¶details: WorkflowDetails
property
readonly
¶info: WorkflowInfo
property
readonly
¶pipeline: Pipeline
property
readonly
¶pipeline_info: PipelineInfo
property
readonly
¶workflow_alias: Optional[str]
property
writable
¶workflow_id: UUID
property
readonly
¶add_step(self, operation, step_id=None, module_config=None, input_connections=None, doc=None, replace_existing=False)
¶Add a step to the workflows current pipeline structure.
If no 'step_id' is provided, a unque one will automatically be generated based on the 'module_type' argument.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
operation |
str |
the module or operation name |
required |
step_id |
Optional[str] |
the id of the new step |
None |
module_config |
Optional[Mapping[str, Any]] |
(optional) configuration for the kiara module this step uses |
None |
input_connections |
Optional[Mapping[str, str]] |
a map with this steps input field name(s) as keys and output field links (format: |
None |
replace_existing |
bool |
if set to 'True', this replaces a step with the same id that already exists, otherwise an exception will be thrown |
False |
Source code in kiara/interfaces/python_api/workflow.py
def add_step(
self,
operation: str,
step_id: Union[str, None] = None,
module_config: Union[None, Mapping[str, Any]] = None,
input_connections: Union[None, Mapping[str, str]] = None,
doc: Union[str, DocumentationMetadataModel, None] = None,
replace_existing: bool = False,
) -> PipelineStep:
"""Add a step to the workflows current pipeline structure.
If no 'step_id' is provided, a unque one will automatically be generated based on the 'module_type' argument.
Arguments:
operation: the module or operation name
step_id: the id of the new step
module_config: (optional) configuration for the kiara module this step uses
input_connections: a map with this steps input field name(s) as keys and output field links (format: <step_id>.<output_field_name>) as value(s).
replace_existing: if set to 'True', this replaces a step with the same id that already exists, otherwise an exception will be thrown
"""
if step_id is None:
step_id = find_free_id(
slugify(operation, separator="_"), current_ids=self._steps.keys()
)
if "." in step_id:
raise Exception(f"Invalid step id '{step_id}': id can't contain '.'.")
if step_id in self._steps.keys() and not replace_existing:
raise Exception(
f"Can't add step with id '{step_id}': step already exists and 'replace_existing' not set."
)
elif step_id in self._steps.keys():
raise NotImplementedError()
manifest = self._kiara.create_manifest(
module_or_operation=operation, config=module_config
)
module = self._kiara.create_module(manifest=manifest)
step = PipelineStep(
step_id=step_id,
module_type=module.module_type_name,
module_config=module.config.dict(),
module_details=KiaraModuleInstance.from_module(module=module),
doc=doc,
)
step._module = module
self._steps[step_id] = step
if input_connections:
for k, v in input_connections.items():
self.connect_to_inputs(v, f"{step_id}.{k}")
self._invalidate_pipeline()
return step
add_steps(self, *pipeline_steps, *, replace_existing=False, clear_existing=False)
¶Source code in kiara/interfaces/python_api/workflow.py
def add_steps(
self,
*pipeline_steps: PipelineStep,
replace_existing: bool = False,
clear_existing: bool = False,
):
if clear_existing:
self.clear_steps()
for step in pipeline_steps:
if step.step_id in self._steps.keys() and not replace_existing:
raise Exception(
f"Can't add step with id '{step.step_id}': step with that id already exists and 'replace_existing' not set."
)
for step in pipeline_steps:
self._steps[step.step_id] = step
self._invalidate_pipeline()
clear_steps(self, *step_ids)
¶Source code in kiara/interfaces/python_api/workflow.py
def clear_steps(self, *step_ids: str):
if not step_ids:
self._steps.clear()
else:
for step_id in step_ids:
self._steps.pop(step_id, None)
self._invalidate_pipeline()
connect_fields(self, *fields)
¶Source code in kiara/interfaces/python_api/workflow.py
def connect_fields(self, *fields: Union[Tuple[str, str], str]):
pairs = []
current_pair = None
for field in fields:
if isinstance(field, str):
tokens = field.split(".")
if not len(tokens) == 2:
raise Exception(
f"Can't connect field '{field}', field name must be in format: <step_id>.<field_name>."
)
if not current_pair:
current_pair = [tokens]
else:
if not len(current_pair) == 1:
raise Exception(
f"Can't connect fields, invalid input(s): {fields}"
)
current_pair.append(tokens)
pairs.append(current_pair)
current_pair = None
else:
if not len(field) == 2:
raise Exception(
f"Can't connect fields, field tuples must have length 2: {field}"
)
if current_pair:
raise Exception(
f"Can't connect fields, dangling single field: {current_pair}"
)
pair = []
for f in field:
tokens = f.split(".")
if not len(tokens) == 2:
raise Exception(
f"Can't connect field '{f}', field name must be in format: <step_id>.<field_name>."
)
pair.append(tokens)
pairs.append(pair)
for pair in pairs:
self.connect_steps(pair[0][0], pair[0][1], pair[1][0], pair[1][1])
connect_steps(self, source_step, source_field, target_step, target_field)
¶Source code in kiara/interfaces/python_api/workflow.py
def connect_steps(
self,
source_step: Union[PipelineStep, str],
source_field: str,
target_step: Union[PipelineStep, str],
target_field: str,
):
if isinstance(source_step, str):
source_step_obj = self.get_step(source_step)
else:
source_step_obj = source_step
if isinstance(target_step, str):
target_step_obj = self.get_step(target_step)
else:
target_step_obj = target_step
source_step_id = source_step_obj.step_id
target_step_id = target_step_obj.step_id
reversed = False
if source_field not in source_step_obj.module.outputs_schema.keys():
reversed = True
if target_field not in target_step_obj.module.inputs_schema.keys():
reversed = True
if reversed:
if target_field not in target_step_obj.module.outputs_schema.keys():
raise Exception(
f"Can't connect steps '{source_step_id}.{source_field}' -> '{target_step_id}.{target_field}': invalid field name(s)."
)
if source_field not in source_step_obj.module.inputs_schema.keys():
raise Exception(
f"Can't connect steps '{source_step_id}.{source_field}' -> '{target_step_id}.{target_field}': invalid field name(s)."
)
else:
if target_field not in target_step_obj.module.inputs_schema.keys():
raise Exception(
f"Can't connect steps '{source_step_id}.{source_field}' -> '{target_step_id}.{target_field}': invalid field name(s)."
)
if source_field not in source_step_obj.module.outputs_schema.keys():
raise Exception(
f"Can't connect steps '{source_step_id}.{source_field}' -> '{target_step_id}.{target_field}': invalid field name(s)."
)
# we rely on the value of input links to always be a dict here
if not reversed:
source_addr = StepValueAddress(
step_id=source_step_id, value_name=source_field
)
target_step_obj.input_links.setdefault(target_field, []).append(source_addr) # type: ignore
else:
source_addr = StepValueAddress(
step_id=target_step_id, value_name=target_field
)
source_step_obj.input_links.setdefault(source_field, []).append(source_addr) # type: ignore
self._invalidate_pipeline()
connect_to_inputs(self, source_field, *input_fields)
¶Source code in kiara/interfaces/python_api/workflow.py
def connect_to_inputs(self, source_field: str, *input_fields: str):
source_tokens = source_field.split(".")
if len(source_tokens) != 2:
raise Exception(
f"Can't add input link(s): invalid format for provided source '{source_field}', must be string with a single '.' to delimit step-id and output field name."
)
source_step = self.get_step(source_tokens[0])
if source_step is None:
raise Exception(
f"Can't add input link(s)': no source step with id '{source_tokens[0]}' exists."
)
if source_tokens[1] not in source_step.module.outputs_schema.keys():
av_fields = ", ".join(source_step.module.outputs_schema.keys())
raise Exception(
f"Can't add input link(s): source step with id '{source_step.step_id}' does not have output field '{source_tokens[1]}'. Available field names: {av_fields}."
)
source_addr = StepValueAddress(
step_id=source_step.step_id, value_name=source_tokens[1]
)
steps = []
for input_field in input_fields:
input_tokens = input_field.split(".")
if len(input_tokens) != 2:
raise Exception(
f"Can't add input link '{input_field}': invalid format, must be string with a single '.' to delimit step-id and field name."
)
step = self.get_step(input_tokens[0])
if step is None:
raise Exception(
f"Can't add input link '{input_field}': no step with id '{input_tokens[0]}' exists."
)
if input_tokens[1] not in step.module.inputs_schema.keys():
av_fields = ", ".join(step.module.inputs_schema.keys())
raise Exception(
f"Can't add input link '{input_field}': step with id '{input_tokens[0]}' does not have input field '{input_tokens[1]}'. Available field names: {av_fields}."
)
steps.append((step, input_tokens[1]))
for s in steps:
step, field_name = s
# we rely on the value of input links to always be a dict here
step.input_links.setdefault(field_name, []).append(source_addr) # type: ignore
self._invalidate_pipeline()
create(alias=None, blueprint=None, doc=None, kiara=None, overwrite_existing=False)
classmethod
¶Source code in kiara/interfaces/python_api/workflow.py
@classmethod
def create(
cls,
alias: Union[None, str] = None,
blueprint: Union[str, None] = None,
doc: Union[None, str, DocumentationMetadataModel] = None,
kiara: Union[None, "Kiara"] = None,
overwrite_existing: bool = False,
):
if kiara is None:
from kiara.context import Kiara
kiara = Kiara.instance()
operation: Union[None, KiaraOperation] = None
if blueprint:
operation = KiaraOperation(kiara=kiara, operation_name=blueprint)
if doc is None:
doc = operation.operation.doc
details = WorkflowDetails(documentation=doc)
workflow_obj = Workflow(kiara=kiara, workflow=details)
if alias:
workflow_obj.workflow_alias = alias
if blueprint:
assert operation
module = operation.operation.module
if isinstance(module.config, PipelineConfig):
config: PipelineConfig = module.config
else:
raise NotImplementedError()
workflow_obj.add_steps(*config.steps)
return workflow_obj
create_renderable(self, **config)
¶Source code in kiara/interfaces/python_api/workflow.py
def create_renderable(self, **config: Any):
if not self._steps:
return "Invalid workflow: no steps set yet."
return self.info.create_renderable(**config)
get_step(self, step_id)
¶Source code in kiara/interfaces/python_api/workflow.py
def get_step(self, step_id: str) -> PipelineStep:
step = self._steps.get(step_id, None)
if step is None:
if self._steps:
msg = f"Available step ids: {', '.join(self._steps.keys())}"
else:
msg = "Workflow does not have any steps (yet)."
raise Exception(f"No step with id '{step_id}' registered. {msg}")
return step
load(workflow)
classmethod
¶Source code in kiara/interfaces/python_api/workflow.py
@classmethod
def load(cls, workflow: Union[uuid.UUID, str]):
pass
load_state(self, workflow_state_id=None)
¶Load a past state.
If no state id is specified, the latest one that was saved will be used.
Returns:
| Type | Description |
|---|---|
Optional[kiara.models.workflow.WorkflowState] |
'None' if no state was loaded, otherwise the relevant 'WorkflowState' instance |
Source code in kiara/interfaces/python_api/workflow.py
def load_state(
self, workflow_state_id: Union[str, None] = None
) -> Union[None, WorkflowState]:
"""Load a past state.
If no state id is specified, the latest one that was saved will be used.
Returns:
'None' if no state was loaded, otherwise the relevant 'WorkflowState' instance
"""
if workflow_state_id is None:
if not self._workflow_details.workflow_states:
return None
else:
workflow_state_id = self._workflow_details.last_state_id
if workflow_state_id is None:
raise Exception(
f"Can't load current state for workflow '{self.workflow_id}': no state available."
)
state = self._state_cache.get(workflow_state_id, None)
if state is not None:
return state
state = self._kiara.workflow_registry.get_workflow_state(
workflow=self.workflow_id, workflow_state_id=workflow_state_id
)
assert workflow_state_id == state.instance_id
self._state_cache[workflow_state_id] = state
self._all_inputs.clear()
self._current_inputs = None
self.clear_steps()
self._invalidate_pipeline()
self.add_steps(*state.steps)
self._pipeline_input_aliases = dict(state.pipeline_config.input_aliases)
self._pipeline_output_aliasess = dict(state.pipeline_config.output_aliases)
self.set_inputs(**state.inputs)
assert self._current_inputs == state.inputs
self._current_outputs = state.pipeline_info.pipeline_details.pipeline_outputs
self._pipeline_info = state.pipeline_info
self._current_state = state
self._current_info = None
return state
process_steps(self, *step_ids)
¶Source code in kiara/interfaces/python_api/workflow.py
def process_steps(self, *step_ids: str):
self.pipeline # noqa
if not step_ids:
output_job_map = self._pipeline_controller.process_pipeline()
else:
job_ids = {}
for step_id in step_ids:
job_id = self._pipeline_controller.process_step(
step_id=step_id, wait=True
)
job_ids[step_id] = job_id
output_job_map = self._pipeline_controller.set_processing_results(
job_ids=job_ids
)
self._job_id_cache.update(output_job_map)
self._current_outputs = self.pipeline.get_current_pipeline_outputs()
self._current_state = None
self._pipeline_info = None
self._current_info = None
set_input(self, field_name, value)
¶Source code in kiara/interfaces/python_api/workflow.py
def set_input(self, field_name: str, value: Any):
self.set_inputs(**{field_name: value})
set_input_alias(self, input_field, alias)
¶Source code in kiara/interfaces/python_api/workflow.py
def set_input_alias(self, input_field: str, alias: str):
self._pipeline_input_aliases[input_field] = alias
self._invalidate_pipeline()
set_inputs(self, **inputs)
¶Source code in kiara/interfaces/python_api/workflow.py
def set_inputs(self, **inputs: Any):
invalid = []
for k, v in inputs.items():
if k not in self.pipeline.structure.pipeline_inputs_schema.keys():
invalid.append(k)
if invalid:
raise Exception(
f"Can't set pipeline inputs, invalid field(s): '{', '.join(invalid)}'. Available inputs: '{', '.join(self.pipeline.structure.pipeline_inputs_schema.keys())}'"
)
changed = False
for k, v in inputs.items():
# TODO: better equality test?
if k == self._all_inputs.get(k, None):
continue
self._all_inputs[k] = v
changed = True
if changed:
self._current_info = None
self._current_state = None
self._current_inputs = None
self._current_outputs = None
self._pipeline_info = None
self._apply_inputs()
set_output_alias(self, output_field, alias)
¶Source code in kiara/interfaces/python_api/workflow.py
def set_output_alias(self, output_field: str, alias: str):
self._pipeline_output_aliasess[output_field] = alias
snapshot(self, save=True)
¶Source code in kiara/interfaces/python_api/workflow.py
def snapshot(self, save: bool = True) -> WorkflowState:
state = self.current_state
if not self._is_stored:
aliases = []
if self._workflow_alias:
aliases.append(self._workflow_alias)
self._kiara.workflow_registry.register_workflow(
workflow_details=self._workflow_details, workflow_aliases=aliases
)
self._is_stored = True
if save:
for value in state.inputs.values():
self._kiara.data_registry.store_value(value=value)
for value in self.current_outputs.values():
if value in [NOT_SET_VALUE_ID, NONE_VALUE_ID]:
continue
self._kiara.data_registry.store_value(value=value)
job_id = self._job_id_cache[value]
try:
self._kiara.job_registry.store_job_record(job_id=job_id)
except Exception as e:
print(e)
self._workflow_details = self._kiara.workflow_registry.add_workflow_state(
workflow=self._workflow_details, workflow_state=state
)
return state
WorkflowPipelineController (SinglePipelineController)
¶A [PipelineController][kiara.models.modules.pipeline.controller.PipelineController] that executes all pipeline steps non-interactively.
This is the default implementation of a PipelineController, and probably the most simple implementation of one.
It waits until all inputs are set, after which it executes all pipeline steps in the required order.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline |
the pipeline to control |
required | |
auto_process |
whether to automatically start processing the pipeline as soon as the input set is valid |
required |
Source code in kiara/interfaces/python_api/workflow.py
class WorkflowPipelineController(SinglePipelineController):
"""A [PipelineController][kiara.models.modules.pipeline.controller.PipelineController] that executes all pipeline steps non-interactively.
This is the default implementation of a ``PipelineController``, and probably the most simple implementation of one.
It waits until all inputs are set, after which it executes all pipeline steps in the required order.
Arguments:
pipeline: the pipeline to control
auto_process: whether to automatically start processing the pipeline as soon as the input set is valid
"""
def __init__(
self,
kiara: "Kiara",
):
self._is_running: bool = False
super().__init__(job_registry=kiara.job_registry)
def _pipeline_event_occurred(self, event: PipelineEvent):
if event.pipeline_id != self.pipeline.pipeline_id:
return
self._pipeline_details = None
def process_pipeline(self) -> Mapping[uuid.UUID, uuid.UUID]:
log = logger.bind(pipeline_id=self.pipeline.pipeline_id)
if self._is_running:
log.debug(
"ignore.pipeline_process",
reason="Pipeline already running.",
)
raise Exception("Pipeline already running.")
log.debug("execute.pipeline")
self._is_running = True
result: Dict[uuid.UUID, uuid.UUID] = {}
try:
for idx, stage in enumerate(
self.pipeline.structure.processing_stages, start=1
):
log.debug(
"execute.pipeline.stage",
stage=idx,
)
job_ids = {}
stage_failed = False
for step_id in stage:
log.debug(
"execute.pipeline.step",
step_id=step_id,
)
try:
job_id = self.process_step(step_id)
job_ids[step_id] = job_id
except Exception as e:
# TODO: cancel running jobs?
log_exception(e)
log.error(
"error.processing.pipeline",
step_id=step_id,
error=e,
)
stage_failed = True
output_job_map = self.set_processing_results(job_ids=job_ids)
result.update(output_job_map)
if not stage_failed:
log.debug(
"execute_finished.pipeline.stage",
stage=idx,
)
else:
log.debug(
"execute_failed.pipeline.stage",
stage=idx,
)
break
except Exception as e:
log_exception(e)
finally:
self._is_running = False
log.debug("execute_finished.pipeline")
return result
process_pipeline(self)
¶Source code in kiara/interfaces/python_api/workflow.py
def process_pipeline(self) -> Mapping[uuid.UUID, uuid.UUID]:
log = logger.bind(pipeline_id=self.pipeline.pipeline_id)
if self._is_running:
log.debug(
"ignore.pipeline_process",
reason="Pipeline already running.",
)
raise Exception("Pipeline already running.")
log.debug("execute.pipeline")
self._is_running = True
result: Dict[uuid.UUID, uuid.UUID] = {}
try:
for idx, stage in enumerate(
self.pipeline.structure.processing_stages, start=1
):
log.debug(
"execute.pipeline.stage",
stage=idx,
)
job_ids = {}
stage_failed = False
for step_id in stage:
log.debug(
"execute.pipeline.step",
step_id=step_id,
)
try:
job_id = self.process_step(step_id)
job_ids[step_id] = job_id
except Exception as e:
# TODO: cancel running jobs?
log_exception(e)
log.error(
"error.processing.pipeline",
step_id=step_id,
error=e,
)
stage_failed = True
output_job_map = self.set_processing_results(job_ids=job_ids)
result.update(output_job_map)
if not stage_failed:
log.debug(
"execute_finished.pipeline.stage",
stage=idx,
)
else:
log.debug(
"execute_failed.pipeline.stage",
stage=idx,
)
break
except Exception as e:
log_exception(e)
finally:
self._is_running = False
log.debug("execute_finished.pipeline")
return result
WorkflowStatus (KiaraModel)
pydantic-model
¶Source code in kiara/interfaces/python_api/workflow.py
class WorkflowStatus(KiaraModel):
pass
tui
special
¶
Modules¶
pager
¶logger
¶
PagerApp (App)
¶Source code in kiara/interfaces/tui/pager.py
class PagerApp(App):
def __init__(self, **kwargs):
self._control = PagerControl()
self._pager = ValuePager()
self._pager._render_op = kwargs.pop("operation")
self._pager._value = kwargs.pop("value")
self._pager._kiara_api = kwargs.pop("kiara_api")
self._pager._control_widget = self._control
self._control._pager = self._pager
super().__init__(**kwargs)
# async def on_mount(self) -> None:
#
# await self.view.dock(Footer(), edge="bottom")
# await self.view.dock(self._pager, name="data")
async def on_mount(self) -> None:
await self.view.dock(self._control, edge="bottom", size=10)
await self.view.dock(self._pager, edge="top")
async def on_load(self, event):
await self.bind("q", "quit", "Quit")
async def on_key(self, event):
self._control.key_pressed(event.key)
on_key(self, event)
async
¶Source code in kiara/interfaces/tui/pager.py
async def on_key(self, event):
self._control.key_pressed(event.key)
on_load(self, event)
async
¶Source code in kiara/interfaces/tui/pager.py
async def on_load(self, event):
await self.bind("q", "quit", "Quit")
on_mount(self)
async
¶Source code in kiara/interfaces/tui/pager.py
async def on_mount(self) -> None:
await self.view.dock(self._control, edge="bottom", size=10)
await self.view.dock(self._pager, edge="top")
PagerControl (Widget)
¶Source code in kiara/interfaces/tui/pager.py
class PagerControl(Widget):
_pager: ValuePager = None # type: ignore
_scene_keys: Dict[str, Union[None, RenderScene]] = None # type: ignore
current_result: RenderValueResult = Reactive(None) # type: ignore
def key_pressed(self, key: str):
if key in self._scene_keys.keys():
new_ri = self._scene_keys.get(key)
if new_ri:
self._pager.update_render_scene(new_ri.dict())
else:
self.log(f"No matching scene for key: {key}")
def get_title(
self,
key: str,
scene: Union[None, RenderScene],
scene_keys: Dict[str, Union[None, RenderScene]],
):
last_token = key.split(".")[-1]
title = None
for idx, command_key in enumerate(last_token):
if command_key not in scene_keys.keys():
title = last_token.replace(command_key, f"\[{command_key}]", 1) # noqa
break
if title is None:
raise NotImplementedError("Could not find free command key.")
scene_keys[command_key] = scene
if scene is None or scene.disabled:
title = f"[grey46]{title}[/grey46]"
return title
def render_sub_command_tree(
self,
key: str,
scene: Union[RenderScene, None],
scene_keys: Dict[str, Union[None, RenderScene]],
forced_titles: Dict[str, str],
node: Union[Tree] = None,
):
if key in forced_titles.keys():
title = forced_titles[key]
else:
title = self.get_title(key=key, scene=scene, scene_keys=scene_keys)
if node is None:
node = Tree(title)
else:
node = node.add(title)
if scene:
for scene_key, sub_scene in scene.related_scenes.items():
self.render_sub_command_tree(
key=f"{key}.{scene_key}",
scene=sub_scene,
scene_keys=scene_keys,
forced_titles=forced_titles,
node=node,
)
return node
def render(self) -> RenderableType:
if self.current_result is None:
return "-- no value --"
scene_keys: Dict[str, Union[None, RenderScene]] = {}
table = Table(show_header=False, box=box.SIMPLE)
row = []
forced_titles = {}
for key, scene in self.current_result.related_scenes.items():
title = self.get_title(key=key, scene=scene, scene_keys=scene_keys)
forced_titles[key] = title
for key, scene in self.current_result.related_scenes.items():
table.add_column(f"category: {key}")
row.append(
self.render_sub_command_tree(
key=key,
scene=scene,
scene_keys=scene_keys,
forced_titles=forced_titles,
)
)
table.add_row(*row)
self._scene_keys = scene_keys
return table
can_focus
¶current_result
¶get_title(self, key, scene, scene_keys)
¶Source code in kiara/interfaces/tui/pager.py
def get_title(
self,
key: str,
scene: Union[None, RenderScene],
scene_keys: Dict[str, Union[None, RenderScene]],
):
last_token = key.split(".")[-1]
title = None
for idx, command_key in enumerate(last_token):
if command_key not in scene_keys.keys():
title = last_token.replace(command_key, f"\[{command_key}]", 1) # noqa
break
if title is None:
raise NotImplementedError("Could not find free command key.")
scene_keys[command_key] = scene
if scene is None or scene.disabled:
title = f"[grey46]{title}[/grey46]"
return title
key_pressed(self, key)
¶Source code in kiara/interfaces/tui/pager.py
def key_pressed(self, key: str):
if key in self._scene_keys.keys():
new_ri = self._scene_keys.get(key)
if new_ri:
self._pager.update_render_scene(new_ri.dict())
else:
self.log(f"No matching scene for key: {key}")
render(self)
¶Get renderable for widget.
Returns:
| Type | Description |
|---|---|
RenderableType |
Any renderable |
Source code in kiara/interfaces/tui/pager.py
def render(self) -> RenderableType:
if self.current_result is None:
return "-- no value --"
scene_keys: Dict[str, Union[None, RenderScene]] = {}
table = Table(show_header=False, box=box.SIMPLE)
row = []
forced_titles = {}
for key, scene in self.current_result.related_scenes.items():
title = self.get_title(key=key, scene=scene, scene_keys=scene_keys)
forced_titles[key] = title
for key, scene in self.current_result.related_scenes.items():
table.add_column(f"category: {key}")
row.append(
self.render_sub_command_tree(
key=key,
scene=scene,
scene_keys=scene_keys,
forced_titles=forced_titles,
)
)
table.add_row(*row)
self._scene_keys = scene_keys
return table
render_sub_command_tree(self, key, scene, scene_keys, forced_titles, node=None)
¶Source code in kiara/interfaces/tui/pager.py
def render_sub_command_tree(
self,
key: str,
scene: Union[RenderScene, None],
scene_keys: Dict[str, Union[None, RenderScene]],
forced_titles: Dict[str, str],
node: Union[Tree] = None,
):
if key in forced_titles.keys():
title = forced_titles[key]
else:
title = self.get_title(key=key, scene=scene, scene_keys=scene_keys)
if node is None:
node = Tree(title)
else:
node = node.add(title)
if scene:
for scene_key, sub_scene in scene.related_scenes.items():
self.render_sub_command_tree(
key=f"{key}.{scene_key}",
scene=sub_scene,
scene_keys=scene_keys,
forced_titles=forced_titles,
node=node,
)
return node
ValuePager (Widget)
¶Source code in kiara/interfaces/tui/pager.py
class ValuePager(Widget):
_render_op: Operation = None # type: ignore
_kiara_api: KiaraAPI = None # type: ignore
_value: Value = None # type: ignore
_control_widget: "PagerControl" = None # type: ignore
render_scene: Union[Mapping[str, Any], None] = Reactive(None) # type: ignore
current_result: Union[RenderValueResult, None] = None # type: ignore
def get_num_rows(self) -> int:
rows = self.size.height - 5
if rows <= 0:
rows = 1
return rows
def update_render_scene(self, new_render_scene: Mapping[str, Any]):
new_ri = dict(new_render_scene)
new_ri.setdefault("render_config", {})["number_of_rows"] = self.get_num_rows()
self.render_scene = new_ri
def render(self) -> RenderableType:
if self._value is None or not self._value.is_set:
return "-- no value --"
render_scene: Dict[str, Any] = self.render_scene # type: ignore
if render_scene is None:
render_scene = {"render_config": {"number_of_rows": self.get_num_rows()}}
render_result = self._kiara_api.render_value(
value=self._value,
target_format="terminal_renderable",
render_config=render_scene["render_config"],
)
self.current_result = render_result
self._control_widget.current_result = self.current_result
return self.current_result.rendered # type: ignore
can_focus
¶current_result
¶render_scene
¶get_num_rows(self)
¶Source code in kiara/interfaces/tui/pager.py
def get_num_rows(self) -> int:
rows = self.size.height - 5
if rows <= 0:
rows = 1
return rows
render(self)
¶Get renderable for widget.
Returns:
| Type | Description |
|---|---|
RenderableType |
Any renderable |
Source code in kiara/interfaces/tui/pager.py
def render(self) -> RenderableType:
if self._value is None or not self._value.is_set:
return "-- no value --"
render_scene: Dict[str, Any] = self.render_scene # type: ignore
if render_scene is None:
render_scene = {"render_config": {"number_of_rows": self.get_num_rows()}}
render_result = self._kiara_api.render_value(
value=self._value,
target_format="terminal_renderable",
render_config=render_scene["render_config"],
)
self.current_result = render_result
self._control_widget.current_result = self.current_result
return self.current_result.rendered # type: ignore
update_render_scene(self, new_render_scene)
¶Source code in kiara/interfaces/tui/pager.py
def update_render_scene(self, new_render_scene: Mapping[str, Any]):
new_ri = dict(new_render_scene)
new_ri.setdefault("render_config", {})["number_of_rows"] = self.get_num_rows()
self.render_scene = new_ri
models
special
¶
Classes¶
KiaraModel (ABC, BaseModel, JupyterMixin)
pydantic-model
¶
Base class that all models in kiara inherit from.
This class provides utility functions for things like rendering the model on terminal or as html, integration into a tree hierarchy of the overall kiara context, hashing, etc.
Source code in kiara/models/__init__.py
class KiaraModel(ABC, BaseModel, JupyterMixin):
"""Base class that all models in kiara inherit from.
This class provides utility functions for things like rendering the model on terminal or as html, integration into
a tree hierarchy of the overall kiara context, hashing, etc.
"""
__slots__ = ["__weakref__"]
class Config(object):
json_loads = orjson.loads
json_dumps = orjson_dumps
extra = Extra.forbid
# @classmethod
# def get_model_title(cls):
#
# return to_camel_case(cls._kiara_model_name)
@classmethod
def get_schema_hash(cls) -> int:
if cls._schema_hash_cache is not None:
return cls._schema_hash_cache
obj = cls.schema_json()
h = DeepHash(obj, hasher=KIARA_HASH_FUNCTION)
cls._schema_hash_cache = h[obj]
return cls._schema_hash_cache
_graph_cache: Union[nx.DiGraph, None] = PrivateAttr(default=None)
_subcomponent_names_cache: Union[List[str], None] = PrivateAttr(default=None)
_dynamic_subcomponents: Dict[str, "KiaraModel"] = PrivateAttr(default_factory=dict)
_id_cache: Union[str, None] = PrivateAttr(default=None)
_category_id_cache: Union[str, None] = PrivateAttr(default=None)
_schema_hash_cache: ClassVar = None
_cid_cache: Union[CID, None] = PrivateAttr(default=None)
_dag_cache: Union[bytes, None] = PrivateAttr(default=None)
_size_cache: Union[int, None] = PrivateAttr(default=None)
def _retrieve_data_to_hash(self) -> EncodableType:
"""Return data important for hashing this model instance. Implemented by sub-classes.
This returns the relevant data that makes this model unique, excluding any secondary metadata that is not
necessary for this model to be used functionally. Like for example documentation.
"""
return self.dict()
@property
def instance_id(self) -> str:
"""The unique id of this model, within its category."""
if self._id_cache is not None:
return self._id_cache
self._id_cache = self._retrieve_id()
return self._id_cache
@property
def instance_cid(self) -> CID:
if self._cid_cache is None:
self._compute_cid()
return self._cid_cache # type: ignore
@property
def instance_dag(self) -> bytes:
if self._dag_cache is None:
self._compute_cid()
return self._dag_cache # type: ignore
@property
def instance_size(self) -> int:
if self._size_cache is None:
self._compute_cid()
return self._size_cache # type: ignore
@property
def model_type_id(self) -> str:
"""The id of the category of this model."""
if hasattr(self.__class__, "_kiara_model_id"):
return self._kiara_model_id # type: ignore
else:
return _default_id_func(self.__class__)
def _retrieve_id(self) -> str:
return str(self.instance_cid)
def _compute_cid(self):
"""A hash for this model."""
if self._cid_cache is not None:
return
obj = self._retrieve_data_to_hash()
dag, cid = compute_cid(data=obj)
self._cid_cache = cid
self._dag_cache = dag
self._size_cache = len(dag)
# ==========================================================================================
# subcomponent related methods
@property
def subcomponent_keys(self) -> Iterable[str]:
"""The keys of available sub-components of this model."""
if self._subcomponent_names_cache is None:
self._subcomponent_names_cache = sorted(self._retrieve_subcomponent_keys())
return self._subcomponent_names_cache
@property
def subcomponent_tree(self) -> Union[nx.DiGraph, None]:
"""A tree structure, containing all sub-components (and their subcomponents) of this model."""
if not self.subcomponent_keys:
return None
if self._graph_cache is None:
self._graph_cache = assemble_subcomponent_graph(self)
return self._graph_cache
def get_subcomponent(self, path: str) -> "KiaraModel":
"""Retrieve the subcomponent identified by the specified path."""
if path not in self._dynamic_subcomponents.keys():
self._dynamic_subcomponents[path] = self._retrieve_subcomponent(path=path)
return self._dynamic_subcomponents[path]
def find_subcomponents(self, category: str) -> Dict[str, "KiaraModel"]:
"""Find and return all subcomponents of this model that are member of the specified category."""
tree = self.subcomponent_tree
if tree is None:
raise Exception(f"No subcomponents found for category: {category}")
result = {}
for node_id, node in tree.nodes.items():
if not hasattr(node["obj"], "get_category_alias"):
raise NotImplementedError()
if category != node["obj"].get_category_alias():
continue
n_id = node_id[9:] # remove the __self__. token
result[n_id] = node["obj"]
return result
def _retrieve_subcomponent_keys(self) -> Iterable[str]:
"""Retrieve the keys of all subcomponents of this model.
Can be overwritten in sub-classes, by default it tries to automatically determine the subcomponents.
"""
return retrieve_data_subcomponent_keys(self)
def _retrieve_subcomponent(self, path: str) -> "KiaraModel":
"""Retrieve the subcomponent under the specified path.
Can be overwritten in sub-classes, by default it tries to automatically determine the subcomponents.
"""
m = get_subcomponent_from_model(self, path=path)
return m
# ==========================================================================================
# model rendering related methods
def create_panel(self, title: str = None, **config: Any) -> Panel:
rend = self.create_renderable(**config)
return Panel(rend, box=box.ROUNDED, title=title, title_align="left")
def create_html(self, **config) -> str:
template_registry = TemplateRegistry.instance()
template = template_registry.get_template_for_model_type(
model_type=self.model_type_id, template_format="html"
)
if template:
try:
result = template.render(instance=self)
return result
except Exception as e:
log_dev_message(
title="html-rendering error",
msg=f"Failed to render html for model '{self.instance_id}' type '{self.model_type_id}': {e}",
)
try:
html = generate_html(item=self, add_header=False)
return html
except Exception as e:
log_dev_message(
title="html-generation error",
msg=f"Failed to generate html for model '{self.instance_id}' type '{self.model_type_id}': {e}",
)
r = self.create_renderable(**config)
mime_bundle = r._repr_mimebundle_(include=[], exclude=[]) # type: ignore
return mime_bundle["text/html"]
def create_renderable(self, **config: Any) -> RenderableType:
from kiara.utils.output import extract_renderable
include = config.get("include", None)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Key", style="i")
table.add_column("Value")
for k in self.__fields__.keys():
if include is not None and k not in include:
continue
attr = getattr(self, k)
v = extract_renderable(attr)
table.add_row(k, v)
return table
def create_renderable_tree(self, **config: Any) -> Tree:
show_data = config.get("show_data", False)
tree = create_subcomponent_tree_renderable(data=self, show_data=show_data)
return tree
def create_info_data(self, **config) -> Mapping[str, Any]:
include = config.get("include", None)
result = {}
for k in self.__fields__.keys():
if include is not None and k not in include:
continue
attr = getattr(self, k)
v = attr
result[k] = v
return result
def as_dict_with_schema(self) -> Dict[str, Dict[str, Any]]:
return {"data": self.dict(), "schema": self.schema()}
def as_json_with_schema(self) -> str:
data_json = self.json()
schema_json = self.schema_json()
return '{"data": ' + data_json + ', "schema": ' + schema_json + "}"
def __hash__(self):
return int.from_bytes(self.instance_cid.digest, "big")
def __eq__(self, other):
if self.__class__ != other.__class__:
return False
else:
return (self.instance_id, self.instance_cid) == (
other.instance_id,
other.instance_cid,
)
def __repr__(self):
try:
model_id = self.instance_id
except Exception:
model_id = "-- n/a --"
return f"{self.__class__.__name__}(model_id={model_id}, category={self.model_type_id}, fields=[{', '.join(self.__fields__.keys())}])"
def __str__(self):
return self.__repr__()
def _repr_html_(self):
return self.create_html()
def __rich_console__(
self, console: Console, options: ConsoleOptions
) -> RenderResult:
yield self.create_renderable()
# def _repr_mimebundle_(
# self: "ConsoleRenderable",
# include: Sequence[str],
# exclude: Sequence[str],
# **kwargs: Any,
# ) -> Dict[str, str]:
#
# console = get_console()
# segments = list(console.render(self, console.options))
# html = _render_segments(segments)
# text = console._render_buffer(segments)
# data = {"text/plain": text, "text/html": html}
# if include:
# data = {k: v for (k, v) in data.items() if k in include}
# if exclude:
# data = {k: v for (k, v) in data.items() if k not in exclude}
# return data
Attributes¶
instance_cid: CID
property
readonly
¶instance_dag: bytes
property
readonly
¶instance_id: str
property
readonly
¶The unique id of this model, within its category.
instance_size: int
property
readonly
¶model_type_id: str
property
readonly
¶The id of the category of this model.
subcomponent_keys: Iterable[str]
property
readonly
¶The keys of available sub-components of this model.
subcomponent_tree: Optional[networkx.classes.digraph.DiGraph]
property
readonly
¶A tree structure, containing all sub-components (and their subcomponents) of this model.
Config
¶Source code in kiara/models/__init__.py
class Config(object):
json_loads = orjson.loads
json_dumps = orjson_dumps
extra = Extra.forbid
extra
¶json_loads
¶json_dumps(v, *, default=None, **args)
¶Source code in kiara/models/__init__.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
Methods¶
as_dict_with_schema(self)
¶Source code in kiara/models/__init__.py
def as_dict_with_schema(self) -> Dict[str, Dict[str, Any]]:
return {"data": self.dict(), "schema": self.schema()}
as_json_with_schema(self)
¶Source code in kiara/models/__init__.py
def as_json_with_schema(self) -> str:
data_json = self.json()
schema_json = self.schema_json()
return '{"data": ' + data_json + ', "schema": ' + schema_json + "}"
create_html(self, **config)
¶Source code in kiara/models/__init__.py
def create_html(self, **config) -> str:
template_registry = TemplateRegistry.instance()
template = template_registry.get_template_for_model_type(
model_type=self.model_type_id, template_format="html"
)
if template:
try:
result = template.render(instance=self)
return result
except Exception as e:
log_dev_message(
title="html-rendering error",
msg=f"Failed to render html for model '{self.instance_id}' type '{self.model_type_id}': {e}",
)
try:
html = generate_html(item=self, add_header=False)
return html
except Exception as e:
log_dev_message(
title="html-generation error",
msg=f"Failed to generate html for model '{self.instance_id}' type '{self.model_type_id}': {e}",
)
r = self.create_renderable(**config)
mime_bundle = r._repr_mimebundle_(include=[], exclude=[]) # type: ignore
return mime_bundle["text/html"]
create_info_data(self, **config)
¶Source code in kiara/models/__init__.py
def create_info_data(self, **config) -> Mapping[str, Any]:
include = config.get("include", None)
result = {}
for k in self.__fields__.keys():
if include is not None and k not in include:
continue
attr = getattr(self, k)
v = attr
result[k] = v
return result
create_panel(self, title=None, **config)
¶Source code in kiara/models/__init__.py
def create_panel(self, title: str = None, **config: Any) -> Panel:
rend = self.create_renderable(**config)
return Panel(rend, box=box.ROUNDED, title=title, title_align="left")
create_renderable(self, **config)
¶Source code in kiara/models/__init__.py
def create_renderable(self, **config: Any) -> RenderableType:
from kiara.utils.output import extract_renderable
include = config.get("include", None)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Key", style="i")
table.add_column("Value")
for k in self.__fields__.keys():
if include is not None and k not in include:
continue
attr = getattr(self, k)
v = extract_renderable(attr)
table.add_row(k, v)
return table
create_renderable_tree(self, **config)
¶Source code in kiara/models/__init__.py
def create_renderable_tree(self, **config: Any) -> Tree:
show_data = config.get("show_data", False)
tree = create_subcomponent_tree_renderable(data=self, show_data=show_data)
return tree
find_subcomponents(self, category)
¶Find and return all subcomponents of this model that are member of the specified category.
Source code in kiara/models/__init__.py
def find_subcomponents(self, category: str) -> Dict[str, "KiaraModel"]:
"""Find and return all subcomponents of this model that are member of the specified category."""
tree = self.subcomponent_tree
if tree is None:
raise Exception(f"No subcomponents found for category: {category}")
result = {}
for node_id, node in tree.nodes.items():
if not hasattr(node["obj"], "get_category_alias"):
raise NotImplementedError()
if category != node["obj"].get_category_alias():
continue
n_id = node_id[9:] # remove the __self__. token
result[n_id] = node["obj"]
return result
get_schema_hash()
classmethod
¶Source code in kiara/models/__init__.py
@classmethod
def get_schema_hash(cls) -> int:
if cls._schema_hash_cache is not None:
return cls._schema_hash_cache
obj = cls.schema_json()
h = DeepHash(obj, hasher=KIARA_HASH_FUNCTION)
cls._schema_hash_cache = h[obj]
return cls._schema_hash_cache
get_subcomponent(self, path)
¶Retrieve the subcomponent identified by the specified path.
Source code in kiara/models/__init__.py
def get_subcomponent(self, path: str) -> "KiaraModel":
"""Retrieve the subcomponent identified by the specified path."""
if path not in self._dynamic_subcomponents.keys():
self._dynamic_subcomponents[path] = self._retrieve_subcomponent(path=path)
return self._dynamic_subcomponents[path]
Modules¶
aliases
special
¶
VALUE_ALIAS_SEPARATOR
¶logger
¶Classes¶
AliasValueMap (ValueMap)
pydantic-model
¶A model class that holds a tree of values and their schemas.
Source code in kiara/models/aliases/__init__.py
class AliasValueMap(ValueMap):
"""A model class that holds a tree of values and their schemas."""
_kiara_model_id = "instance.value_map.aliases"
alias: Union[str, None] = Field(description="This maps own (full) alias.")
version: int = Field(description="The version of this map (in this maps parent).")
created: Union[datetime.datetime, None] = Field(
description="The time this map was created."
)
assoc_schema: Union[ValueSchema, None] = Field(
description="The schema for this maps associated value."
)
assoc_value: Union[uuid.UUID, None] = Field(
description="The value that is associated with this map."
)
value_items: Dict[str, Dict[int, "AliasValueMap"]] = Field(
description="The values contained in this set.", default_factory=dict
)
_data_registry: "DataRegistry" = PrivateAttr(default=None)
_schema_locked: bool = PrivateAttr(default=False)
_auto_schema: bool = PrivateAttr(default=True)
_is_stored: bool = PrivateAttr(default=False)
def _retrieve_data_to_hash(self) -> Any:
raise NotImplementedError()
@property
def is_stored(self) -> bool:
return self._is_stored
def get_child_map(
self, field_name: str, version: Union[str, None] = None
) -> Union["AliasValueMap", None]:
"""Get the child map for the specified field / version combination.
Raises an error if the child field does not exist. Returns 'None' if not value is set yet (but schema is).
"""
if version is not None:
raise NotImplementedError()
if VALUE_ALIAS_SEPARATOR not in field_name:
if self.values_schema.get(field_name, None) is None:
if not self.values_schema:
msg = "No available fields"
else:
msg = "Available fields: " + ", ".join(self.values_schema.keys())
raise KeyError(f"No field name '{field_name}'. {msg}")
field_items = self.value_items[field_name]
if not field_items:
return None
max_version = max(field_items.keys())
item = field_items[max_version]
return item
else:
child, rest = field_name.split(VALUE_ALIAS_SEPARATOR, maxsplit=1)
if child not in self.values_schema.keys():
raise Exception(
f"No field name '{child}'. Available fields: {', '.join(self.values_schema.keys())}"
)
child_map = self.get_child_map(child)
assert child_map is not None
return child_map.get_child_map(rest)
def get_value_obj(self, field_name: str) -> Value:
item = self.get_child_map(field_name=field_name)
if item is None:
return self._data_registry.NONE_VALUE
if item.assoc_value is None:
raise Exception(f"No value associated for field '{field_name}'.")
return self._data_registry.get_value(value=item.assoc_value)
def get_value_id(self, field_name: str) -> uuid.UUID:
item = self.get_child_map(field_name=field_name)
if item is None:
result = NONE_VALUE_ID
else:
result = item.assoc_value if item.assoc_value is not None else NONE_VALUE_ID
return result
def get_all_value_ids(
self,
) -> Dict[str, uuid.UUID]:
result: Dict[str, uuid.UUID] = {}
for k in self.values_schema.keys():
v_id = self.get_value_id(field_name=k)
if v_id is None:
v_id = NONE_VALUE_ID
result[k] = v_id
return result
def set_alias_schema(self, alias: str, schema: ValueSchema):
if self._schema_locked:
raise Exception(f"Can't add schema for alias '{alias}': schema locked.")
if VALUE_ALIAS_SEPARATOR not in alias:
self._set_local_field_schema(field_name=alias, schema=schema)
else:
child, rest = alias.split(VALUE_ALIAS_SEPARATOR, maxsplit=1)
if child in self.values_schema.keys():
child_map = self.get_child_map(child)
else:
self._set_local_field_schema(
field_name=child, schema=ValueSchema(type="none")
)
child_map = self._set_alias(alias=child, data=None)
assert child_map is not None
child_map.set_alias_schema(alias=rest, schema=schema)
def _set_local_field_schema(self, field_name: str, schema: ValueSchema):
assert field_name is not None
if VALUE_ALIAS_SEPARATOR in field_name:
raise Exception(
f"Can't add schema, field name has alias separator in name: {field_name}. This is most likely a bug."
)
if field_name in self.values_schema.keys():
raise Exception(
f"Can't set alias schema for '{field_name}' to map: schema already set."
)
try:
items = self.get_child_map(field_name)
if items is not None:
raise Exception(
f"Can't set schema for field '{field_name}': already at least one child set for this field."
)
except KeyError:
pass
self.values_schema[field_name] = schema
self.value_items[field_name] = {}
def get_alias(self, alias: str) -> Union["AliasValueMap", None]:
if VALUE_ALIAS_SEPARATOR not in alias:
if "@" in alias:
raise NotImplementedError()
child_map = self.get_child_map(alias)
if child_map is None:
return None
return child_map
else:
child, rest = alias.split(VALUE_ALIAS_SEPARATOR, maxsplit=1)
if "@" in child:
raise NotImplementedError()
child_map = self.get_child_map(field_name=child)
if child_map is None:
return None
return child_map.get_alias(rest)
def set_value(self, field_name: str, data: Any) -> None:
assert VALUE_ALIAS_SEPARATOR not in field_name
self._set_alias(alias=field_name, data=data)
def _set_aliases(self, **aliases: Any) -> Mapping[str, "AliasValueMap"]:
result = {}
for k, v in aliases.items():
r = self._set_alias(alias=k, data=v)
result[k] = r
return result
def _set_alias(self, alias: str, data: Any) -> "AliasValueMap":
if VALUE_ALIAS_SEPARATOR not in alias:
field_name: Union[str, None] = alias
# means we are setting the alias in this map
assert field_name is not None
vs = self.values_schema[alias]
if vs.type == "none":
assert data is None
value_id = None
else:
if data in [None, SpecialValue.NO_VALUE, SpecialValue.NOT_SET]:
if vs.default:
if callable(vs.default):
data = vs.default()
else:
data = copy.deepcopy(vs.default)
value = self._data_registry.register_data(data=data, schema=vs)
value_id = value.value_id
new_map = self._set_local_value_item(
field_name=field_name, value_id=value_id
)
return new_map
else:
child, rest = alias.split(VALUE_ALIAS_SEPARATOR, maxsplit=1)
field_name = None
# means we are dealing with an intermediate alias map
assert rest is not None
assert child is not None
assert field_name is None
if child not in self.value_items.keys():
if not self._auto_schema:
raise Exception(
f"Can't set alias '{alias}', no schema set for field: '{child}'."
)
else:
self.set_alias_schema(alias=child, schema=ValueSchema(type="any"))
field_item: Union[AliasValueMap, None] = None
try:
field_item = self.get_child_map(field_name=child)
except KeyError:
pass
if self.alias:
new_alias = f"{self.alias}.{child}"
else:
new_alias = child
if field_item is None:
new_version = 0
schemas = {}
self.value_items[child] = {}
else:
max_version = len(field_item.keys())
new_version = max_version + 1
assert field_item.alias == new_alias
assert field_item.version == max_version
schemas = field_item.values_schema
new_map = AliasValueMap(
alias=new_alias,
version=new_version,
assoc_schema=self.values_schema[child],
assoc_value=None,
values_schema=schemas,
)
new_map._data_registry = self._data_registry
self.value_items[child][new_version] = new_map
new_map._set_alias(alias=rest, data=data)
return new_map
def _set_local_value_item(
self, field_name: str, value_id: Union[uuid.UUID, None] = None
) -> "AliasValueMap":
assert VALUE_ALIAS_SEPARATOR not in field_name
value: Union[Value, None] = None
if value_id is not None:
value = self._data_registry.get_value(value=value_id)
assert value is not None
assert value.value_id == value_id
if field_name not in self.values_schema.keys():
if not self._auto_schema:
raise Exception(
f"Can't add value for field '{field_name}': field not in schema."
)
else:
if value_id is None:
value_schema = ValueSchema(type="none")
else:
value_schema = value.value_schema # type: ignore
self.set_alias_schema(alias=field_name, schema=value_schema)
field_items = self.value_items.get(field_name, None)
if not field_items:
assert field_items is not None
new_version = 0
values_schema = {}
else:
max_version = max(field_items.keys())
current_map = field_items[max_version]
if value_id == current_map.assoc_value:
logger.debug(
"set_field.skip",
value_id=None,
reason=f"Same value id: {value_id}",
)
return current_map
# TODO: check schema
new_version = max(field_items.keys()) + 1
values_schema = current_map.values_schema
if self.alias:
new_alias = f"{self.alias}.{field_name}"
else:
new_alias = field_name
new_map = AliasValueMap(
alias=new_alias,
version=new_version,
assoc_schema=self.values_schema[field_name],
assoc_value=value_id,
values_schema=values_schema,
)
new_map._data_registry = self._data_registry
self.value_items[field_name][new_version] = new_map
return new_map
def print_tree(self):
t = self.get_tree("base")
terminal_print(t)
def get_tree(self, base_name: str) -> Tree:
if self.assoc_schema:
type_name = self.assoc_schema.type
else:
type_name = "none"
if type_name == "none":
type_str = ""
else:
type_str = f" ({type_name})"
tree = Tree(f"{base_name}{type_str}")
if self.assoc_value:
data = tree.add("__data__")
value = self._data_registry.get_value(self.assoc_value)
data.add(str(value.data))
for field_name, schema in self.values_schema.items():
alias = self.get_alias(alias=field_name)
if alias is not None:
tree.add(alias.get_tree(base_name=field_name))
else:
if schema.type == "none":
type_str = ""
else:
type_str = f" ({schema.type})"
tree.add(f"{field_name}{type_str}")
return tree
def __repr__(self):
return f"AliasMap(assoc_value={self.assoc_value}, field_names={self.value_items.keys()})"
def __str__(self):
return self.__repr__()
alias: str
pydantic-field
¶This maps own (full) alias.
assoc_schema: ValueSchema
pydantic-field
¶The schema for this maps associated value.
assoc_value: UUID
pydantic-field
¶The value that is associated with this map.
created: datetime
pydantic-field
¶The time this map was created.
is_stored: bool
property
readonly
¶value_items: Dict[str, Dict[int, AliasValueMap]]
pydantic-field
¶The values contained in this set.
version: int
pydantic-field
required
¶The version of this map (in this maps parent).
get_alias(self, alias)
¶Source code in kiara/models/aliases/__init__.py
def get_alias(self, alias: str) -> Union["AliasValueMap", None]:
if VALUE_ALIAS_SEPARATOR not in alias:
if "@" in alias:
raise NotImplementedError()
child_map = self.get_child_map(alias)
if child_map is None:
return None
return child_map
else:
child, rest = alias.split(VALUE_ALIAS_SEPARATOR, maxsplit=1)
if "@" in child:
raise NotImplementedError()
child_map = self.get_child_map(field_name=child)
if child_map is None:
return None
return child_map.get_alias(rest)
get_all_value_ids(self)
¶Source code in kiara/models/aliases/__init__.py
def get_all_value_ids(
self,
) -> Dict[str, uuid.UUID]:
result: Dict[str, uuid.UUID] = {}
for k in self.values_schema.keys():
v_id = self.get_value_id(field_name=k)
if v_id is None:
v_id = NONE_VALUE_ID
result[k] = v_id
return result
get_child_map(self, field_name, version=None)
¶Get the child map for the specified field / version combination.
Raises an error if the child field does not exist. Returns 'None' if not value is set yet (but schema is).
Source code in kiara/models/aliases/__init__.py
def get_child_map(
self, field_name: str, version: Union[str, None] = None
) -> Union["AliasValueMap", None]:
"""Get the child map for the specified field / version combination.
Raises an error if the child field does not exist. Returns 'None' if not value is set yet (but schema is).
"""
if version is not None:
raise NotImplementedError()
if VALUE_ALIAS_SEPARATOR not in field_name:
if self.values_schema.get(field_name, None) is None:
if not self.values_schema:
msg = "No available fields"
else:
msg = "Available fields: " + ", ".join(self.values_schema.keys())
raise KeyError(f"No field name '{field_name}'. {msg}")
field_items = self.value_items[field_name]
if not field_items:
return None
max_version = max(field_items.keys())
item = field_items[max_version]
return item
else:
child, rest = field_name.split(VALUE_ALIAS_SEPARATOR, maxsplit=1)
if child not in self.values_schema.keys():
raise Exception(
f"No field name '{child}'. Available fields: {', '.join(self.values_schema.keys())}"
)
child_map = self.get_child_map(child)
assert child_map is not None
return child_map.get_child_map(rest)
get_tree(self, base_name)
¶Source code in kiara/models/aliases/__init__.py
def get_tree(self, base_name: str) -> Tree:
if self.assoc_schema:
type_name = self.assoc_schema.type
else:
type_name = "none"
if type_name == "none":
type_str = ""
else:
type_str = f" ({type_name})"
tree = Tree(f"{base_name}{type_str}")
if self.assoc_value:
data = tree.add("__data__")
value = self._data_registry.get_value(self.assoc_value)
data.add(str(value.data))
for field_name, schema in self.values_schema.items():
alias = self.get_alias(alias=field_name)
if alias is not None:
tree.add(alias.get_tree(base_name=field_name))
else:
if schema.type == "none":
type_str = ""
else:
type_str = f" ({schema.type})"
tree.add(f"{field_name}{type_str}")
return tree
get_value_id(self, field_name)
¶Source code in kiara/models/aliases/__init__.py
def get_value_id(self, field_name: str) -> uuid.UUID:
item = self.get_child_map(field_name=field_name)
if item is None:
result = NONE_VALUE_ID
else:
result = item.assoc_value if item.assoc_value is not None else NONE_VALUE_ID
return result
get_value_obj(self, field_name)
¶Source code in kiara/models/aliases/__init__.py
def get_value_obj(self, field_name: str) -> Value:
item = self.get_child_map(field_name=field_name)
if item is None:
return self._data_registry.NONE_VALUE
if item.assoc_value is None:
raise Exception(f"No value associated for field '{field_name}'.")
return self._data_registry.get_value(value=item.assoc_value)
print_tree(self)
¶Source code in kiara/models/aliases/__init__.py
def print_tree(self):
t = self.get_tree("base")
terminal_print(t)
set_alias_schema(self, alias, schema)
¶Source code in kiara/models/aliases/__init__.py
def set_alias_schema(self, alias: str, schema: ValueSchema):
if self._schema_locked:
raise Exception(f"Can't add schema for alias '{alias}': schema locked.")
if VALUE_ALIAS_SEPARATOR not in alias:
self._set_local_field_schema(field_name=alias, schema=schema)
else:
child, rest = alias.split(VALUE_ALIAS_SEPARATOR, maxsplit=1)
if child in self.values_schema.keys():
child_map = self.get_child_map(child)
else:
self._set_local_field_schema(
field_name=child, schema=ValueSchema(type="none")
)
child_map = self._set_alias(alias=child, data=None)
assert child_map is not None
child_map.set_alias_schema(alias=rest, schema=schema)
set_value(self, field_name, data)
¶Source code in kiara/models/aliases/__init__.py
def set_value(self, field_name: str, data: Any) -> None:
assert VALUE_ALIAS_SEPARATOR not in field_name
self._set_alias(alias=field_name, data=data)
archives
¶
Classes¶
ArchiveGroupInfo (InfoItemGroup)
pydantic-model
¶Source code in kiara/models/archives.py
class ArchiveGroupInfo(InfoItemGroup):
_kiara_model_id = "info.archives"
@classmethod
def base_info_class(cls) -> Type[ItemInfo]:
return ArchiveInfo
@classmethod
def create_from_context(
cls, kiara: "Kiara", group_title: Union[str, None] = None
) -> "ArchiveGroupInfo":
archives = {}
for archive, aliases in kiara.get_all_archives().items():
archives[str(archive.archive_id)] = ArchiveInfo.create_from_archive(
kiara=kiara, archive=archive, archive_aliases=aliases
)
info = cls(group_title=group_title, item_infos=archives)
return info
item_infos: Mapping[str, ArchiveInfo] = Field(
description="The info for each archive."
)
@property
def combined_size(self) -> int:
combined = 0
for archive_info in self.item_infos.values():
size = archive_info.details.size
if size and size > 0:
combined = combined + size
return combined
def create_renderable(self, **config: Any) -> RenderableType:
show_archive_id = config.get("show_archive_id", False)
show_config = config.get("show_config", True)
show_details = config.get("show_details", False)
# by_type: Dict[str, Dict[str, ArchiveInfo]] = {}
# for archive_id, archive in sorted(self.item_infos.items()):
# for item_type in archive.archive_type_info.supported_item_types:
# by_type.setdefault(item_type, {})[archive.type_name] = archive
table = Table(show_header=True, box=box.SIMPLE)
if show_archive_id:
table.add_column("archive id")
table.add_column("alias(es)", style="i")
table.add_column("item type(s)", style="i")
if show_config:
table.add_column("config")
if show_details:
table.add_column("details")
for archive in self.item_infos.values():
row: List[RenderableType] = []
if show_archive_id:
row.append(str(archive.archive_id))
row.append("\n".join(archive.aliases))
row.append("\n".join(archive.archive_type_info.supported_item_types))
if show_config:
config_json = Syntax(
orjson_dumps(archive.config, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
row.append(config_json)
if show_details:
details_json = Syntax(
orjson_dumps(archive.details, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
row.append(details_json)
table.add_row(*row)
return table
combined_size: int
property
readonly
¶base_info_class()
classmethod
¶Source code in kiara/models/archives.py
@classmethod
def base_info_class(cls) -> Type[ItemInfo]:
return ArchiveInfo
create_from_context(kiara, group_title=None)
classmethod
¶Source code in kiara/models/archives.py
@classmethod
def create_from_context(
cls, kiara: "Kiara", group_title: Union[str, None] = None
) -> "ArchiveGroupInfo":
archives = {}
for archive, aliases in kiara.get_all_archives().items():
archives[str(archive.archive_id)] = ArchiveInfo.create_from_archive(
kiara=kiara, archive=archive, archive_aliases=aliases
)
info = cls(group_title=group_title, item_infos=archives)
return info
create_renderable(self, **config)
¶Source code in kiara/models/archives.py
def create_renderable(self, **config: Any) -> RenderableType:
show_archive_id = config.get("show_archive_id", False)
show_config = config.get("show_config", True)
show_details = config.get("show_details", False)
# by_type: Dict[str, Dict[str, ArchiveInfo]] = {}
# for archive_id, archive in sorted(self.item_infos.items()):
# for item_type in archive.archive_type_info.supported_item_types:
# by_type.setdefault(item_type, {})[archive.type_name] = archive
table = Table(show_header=True, box=box.SIMPLE)
if show_archive_id:
table.add_column("archive id")
table.add_column("alias(es)", style="i")
table.add_column("item type(s)", style="i")
if show_config:
table.add_column("config")
if show_details:
table.add_column("details")
for archive in self.item_infos.values():
row: List[RenderableType] = []
if show_archive_id:
row.append(str(archive.archive_id))
row.append("\n".join(archive.aliases))
row.append("\n".join(archive.archive_type_info.supported_item_types))
if show_config:
config_json = Syntax(
orjson_dumps(archive.config, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
row.append(config_json)
if show_details:
details_json = Syntax(
orjson_dumps(archive.details, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
row.append(details_json)
table.add_row(*row)
return table
ArchiveInfo (ItemInfo)
pydantic-model
¶Source code in kiara/models/archives.py
class ArchiveInfo(ItemInfo):
@classmethod
def base_instance_class(cls) -> Type[KiaraArchive]:
return KiaraArchive
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: KiaraArchive, **kwargs):
return cls.create_from_archive(kiara=kiara, archive=instance, **kwargs)
@classmethod
def create_from_archive(
cls,
kiara: "Kiara",
archive: KiaraArchive,
archive_aliases: Union[Iterable[str], None] = None,
):
archive_type_info = ArchiveTypeInfo.create_from_type_class(
archive.__class__, kiara=kiara
)
if archive_aliases is None:
archive_aliases = []
else:
archive_aliases = list(archive_aliases)
return ArchiveInfo(
archive_type_info=archive_type_info,
type_name=str(archive.archive_id),
documentation=archive_type_info.documentation,
authors=archive_type_info.authors,
context=archive_type_info.context,
archive_id=archive.archive_id,
details=archive.get_archive_details(),
config=archive.config.dict(),
aliases=archive_aliases,
)
@classmethod
def category_name(cls) -> str:
return "info.archive"
archive_id: uuid.UUID = Field(description="The (globally unique) archive id.")
archive_type_info: ArchiveTypeInfo = Field(
description="Information about this archives' type."
)
config: Mapping[str, Any] = Field(description="The configuration of this archive.")
details: ArchiveDetails = Field(
description="Type dependent (runtime) details for this archive."
)
aliases: List[str] = Field(
description="Aliases for this archive.", default_factory=list
)
aliases: List[str]
pydantic-field
¶Aliases for this archive.
archive_id: UUID
pydantic-field
required
¶The (globally unique) archive id.
archive_type_info: ArchiveTypeInfo
pydantic-field
required
¶Information about this archives' type.
config: Mapping[str, Any]
pydantic-field
required
¶The configuration of this archive.
details: ArchiveDetails
pydantic-field
required
¶Type dependent (runtime) details for this archive.
base_instance_class()
classmethod
¶Source code in kiara/models/archives.py
@classmethod
def base_instance_class(cls) -> Type[KiaraArchive]:
return KiaraArchive
category_name()
classmethod
¶Source code in kiara/models/archives.py
@classmethod
def category_name(cls) -> str:
return "info.archive"
create_from_archive(kiara, archive, archive_aliases=None)
classmethod
¶Source code in kiara/models/archives.py
@classmethod
def create_from_archive(
cls,
kiara: "Kiara",
archive: KiaraArchive,
archive_aliases: Union[Iterable[str], None] = None,
):
archive_type_info = ArchiveTypeInfo.create_from_type_class(
archive.__class__, kiara=kiara
)
if archive_aliases is None:
archive_aliases = []
else:
archive_aliases = list(archive_aliases)
return ArchiveInfo(
archive_type_info=archive_type_info,
type_name=str(archive.archive_id),
documentation=archive_type_info.documentation,
authors=archive_type_info.authors,
context=archive_type_info.context,
archive_id=archive.archive_id,
details=archive.get_archive_details(),
config=archive.config.dict(),
aliases=archive_aliases,
)
create_from_instance(kiara, instance, **kwargs)
classmethod
¶Source code in kiara/models/archives.py
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: KiaraArchive, **kwargs):
return cls.create_from_archive(kiara=kiara, archive=instance, **kwargs)
ArchiveTypeClassesInfo (TypeInfoItemGroup)
pydantic-model
¶Source code in kiara/models/archives.py
class ArchiveTypeClassesInfo(TypeInfoItemGroup):
_kiara_model_id = "info.archive_types"
@classmethod
def base_info_class(cls) -> Type[ArchiveTypeInfo]:
return ArchiveTypeInfo
type_name: Literal["archive_type"] = "archive_type"
item_infos: Mapping[str, ArchiveTypeInfo] = Field( # type: ignore
description="The archive info instances for each type."
)
ArchiveTypeInfo (TypeInfo)
pydantic-model
¶Source code in kiara/models/archives.py
class ArchiveTypeInfo(TypeInfo):
_kiara_model_id = "info.archive_type"
@classmethod
def create_from_type_class(
self, type_cls: Type[KiaraArchive], kiara: "Kiara"
) -> "ArchiveTypeInfo":
authors_md = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
python_class = PythonClass.from_class(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
type_name = type_cls._archive_type_name # type: ignore
return ArchiveTypeInfo.construct(
type_name=type_name,
documentation=doc,
authors=authors_md,
context=properties_md,
python_class=python_class,
supported_item_types=list(type_cls.supported_item_types()),
)
@classmethod
def base_class(self) -> Type[KiaraArchive]:
return KiaraArchive
@classmethod
def category_name(cls) -> str:
return "archive_type"
is_writable: bool = Field(
description="Whether this archive is writeable.", default=False
)
supported_item_types: List[str] = Field(
description="The item types this archive suports."
)
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
table.add_row("Python class", self.python_class.create_renderable())
table.add_row("is_writeable", "yes" if self.is_writable else "no")
table.add_row(
"supported_item_types", ", ".join(sorted(self.supported_item_types))
)
return table
is_writable: bool
pydantic-field
¶Whether this archive is writeable.
supported_item_types: List[str]
pydantic-field
required
¶The item types this archive suports.
base_class()
classmethod
¶Source code in kiara/models/archives.py
@classmethod
def base_class(self) -> Type[KiaraArchive]:
return KiaraArchive
category_name()
classmethod
¶Source code in kiara/models/archives.py
@classmethod
def category_name(cls) -> str:
return "archive_type"
create_from_type_class(type_cls, kiara)
classmethod
¶Source code in kiara/models/archives.py
@classmethod
def create_from_type_class(
self, type_cls: Type[KiaraArchive], kiara: "Kiara"
) -> "ArchiveTypeInfo":
authors_md = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
python_class = PythonClass.from_class(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
type_name = type_cls._archive_type_name # type: ignore
return ArchiveTypeInfo.construct(
type_name=type_name,
documentation=doc,
authors=authors_md,
context=properties_md,
python_class=python_class,
supported_item_types=list(type_cls.supported_item_types()),
)
create_renderable(self, **config)
¶Source code in kiara/models/archives.py
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
table.add_row("Python class", self.python_class.create_renderable())
table.add_row("is_writeable", "yes" if self.is_writable else "no")
table.add_row(
"supported_item_types", ", ".join(sorted(self.supported_item_types))
)
return table
context
¶
Classes¶
ContextSummaries (BaseModel)
pydantic-model
¶Source code in kiara/models/context.py
class ContextSummaries(BaseModel):
__root__: Dict[str, ContextSummary]
@classmethod
def create_context_summaries(
cls, contexts: Union[Mapping[str, "KiaraContextConfig"], None] = None
):
if not contexts:
kc = KiaraConfig()
contexts = kc.context_configs
return ContextSummaries(
__root__={
a: ContextSummary.create_from_context_config(c, context_name=a)
for a, c in contexts.items()
}
)
def create_renderable(self, **config: Any) -> RenderableType:
full_details = config.get("full_details", False)
if not full_details:
table = Table(box=box.SIMPLE, show_header=True, show_lines=False)
table.add_column("context name", style="i")
table.add_column("context id", style="i")
table.add_column("size on disk")
table.add_column("size of all values")
table.add_column("no. values")
table.add_column("no. aliaes")
for context_name, context_summary in self.__root__.items():
size_on_disk = context_summary.archives.combined_size
value_summary = context_summary.value_summary()
size = humanfriendly.format_size(value_summary["size"])
no_values = str(value_summary["no_values"])
no_aliases = str(len(context_summary.aliases))
table.add_row(
context_name,
str(context_summary.kiara_id),
humanfriendly.format_size(size_on_disk),
size,
no_values,
no_aliases,
)
else:
table = Table(box=box.MINIMAL, show_header=True, show_lines=True)
table.add_column("context_name", style="i")
table.add_column("details")
for context_name, context_summary in self.__root__.items():
table.add_row(context_name, context_summary.create_renderable(**config))
return table
create_context_summaries(contexts=None)
classmethod
¶Source code in kiara/models/context.py
@classmethod
def create_context_summaries(
cls, contexts: Union[Mapping[str, "KiaraContextConfig"], None] = None
):
if not contexts:
kc = KiaraConfig()
contexts = kc.context_configs
return ContextSummaries(
__root__={
a: ContextSummary.create_from_context_config(c, context_name=a)
for a, c in contexts.items()
}
)
create_renderable(self, **config)
¶Source code in kiara/models/context.py
def create_renderable(self, **config: Any) -> RenderableType:
full_details = config.get("full_details", False)
if not full_details:
table = Table(box=box.SIMPLE, show_header=True, show_lines=False)
table.add_column("context name", style="i")
table.add_column("context id", style="i")
table.add_column("size on disk")
table.add_column("size of all values")
table.add_column("no. values")
table.add_column("no. aliaes")
for context_name, context_summary in self.__root__.items():
size_on_disk = context_summary.archives.combined_size
value_summary = context_summary.value_summary()
size = humanfriendly.format_size(value_summary["size"])
no_values = str(value_summary["no_values"])
no_aliases = str(len(context_summary.aliases))
table.add_row(
context_name,
str(context_summary.kiara_id),
humanfriendly.format_size(size_on_disk),
size,
no_values,
no_aliases,
)
else:
table = Table(box=box.MINIMAL, show_header=True, show_lines=True)
table.add_column("context_name", style="i")
table.add_column("details")
for context_name, context_summary in self.__root__.items():
table.add_row(context_name, context_summary.create_renderable(**config))
return table
ContextSummary (KiaraModel)
pydantic-model
¶Source code in kiara/models/context.py
class ContextSummary(KiaraModel):
@classmethod
def create_from_context_config(
cls,
config: KiaraContextConfig,
context_name: Union[str, None] = None,
runtime_config: Union[KiaraRuntimeConfig, None] = None,
):
from kiara.context import Kiara
kiara = Kiara(config=config, runtime_config=runtime_config)
return cls.create_from_context(kiara=kiara, context_name=context_name)
@classmethod
def create_from_context(cls, kiara: "Kiara", context_name: Union[str, None] = None):
value_ids = list(kiara.data_registry.retrieve_all_available_value_ids())
aliases = {
a.full_alias: a.value_id for a in kiara.alias_registry.aliases.values()
}
archives_info = ArchiveGroupInfo.create_from_context(kiara=kiara)
result = ContextSummary.construct(
kiara_id=kiara.id,
value_ids=value_ids,
aliases=aliases,
context_name=context_name,
archives=archives_info,
)
result._kiara = kiara
return result
kiara_id: uuid.UUID = Field(
description="The (globally unique) id of the kiara context."
)
context_name: Union[str, None] = Field(
description="The local alias for this context."
)
value_ids: List[uuid.UUID] = Field(
description="The ids of all stored values in this context."
)
aliases: Dict[str, uuid.UUID] = Field(
description="All available aliases within this context (and the value ids they refer to)."
)
archives: ArchiveGroupInfo = Field(
description="The archives registered in this context."
)
_kiara: Union["Kiara", None] = PrivateAttr()
@property
def kiara_context(self) -> "Kiara":
if self._kiara is None:
raise Exception("Kiara context object not set.")
return self._kiara
def value_summary(self) -> Dict[str, Any]:
sum_size = 0
types: Dict[str, int] = {}
internal_types: Dict[str, int] = {}
no_of_values = len(self.value_ids)
for value_id in self.value_ids:
value = self.kiara_context.data_registry.get_value(value=value_id)
sum_size = sum_size + value.value_size
if self.kiara_context.type_registry.is_internal_type(value.data_type_name):
if value.data_type_name not in internal_types.keys():
internal_types[value.data_type_name] = 1
else:
internal_types[value.data_type_name] += 1
else:
if value.data_type_name not in types.keys():
types[value.data_type_name] = 1
else:
types[value.data_type_name] += 1
types.setdefault(value.data_type_name, 0)
return {
"size": sum_size,
"no_values": no_of_values,
"types": types,
"internal_types": internal_types,
}
def alias_summary(self) -> Dict[str, Any]:
sum_size = 0
types: Dict[str, int] = {}
internal_types: Dict[str, int] = {}
no_of_values = len(self.value_ids)
for alias, value_id in self.aliases.items():
value = self.kiara_context.data_registry.get_value(value=value_id)
sum_size = sum_size + value.value_size
if self.kiara_context.type_registry.is_internal_type(value.data_type_name):
if value.data_type_name not in internal_types.keys():
internal_types[value.data_type_name] = 1
else:
internal_types[value.data_type_name] += 1
else:
if value.data_type_name not in types.keys():
types[value.data_type_name] = 1
else:
types[value.data_type_name] += 1
types.setdefault(value.data_type_name, 0)
return {
"size": sum_size,
"no_values": no_of_values,
"types": types,
"internal_types": internal_types,
}
def create_renderable(self, **config: Any) -> RenderableType:
full_details = config.get("full_details", False)
show_value_ids = config.get("show_value_ids", False)
show_archive_info = config.get("show_archive_info", True)
table = Table(box=box.SIMPLE, show_header=False)
table.add_column("Property", style="i")
table.add_column("Value")
if self.context_name:
table.add_row("context name", self.context_name)
table.add_row("kiara_id", str(self.kiara_id))
size_on_disk = humanfriendly.format_size(self.archives.combined_size)
table.add_row("size on disk", size_on_disk)
value_sum = self.value_summary()
v_table = Table(box=box.SIMPLE, show_header=False)
v_table.add_column("Property")
v_table.add_column("Value")
v_table.add_row("no. values", str(value_sum["no_values"]))
v_table.add_row("combined size", format_size(value_sum["size"]))
if full_details and show_value_ids:
if self.value_ids:
value_ids = sorted((str(v) for v in self.value_ids))
v_table.add_row("value_ids", value_ids[0])
for v_id in value_ids[1:]:
v_table.add_row("", v_id)
else:
v_table.add_row("value_ids", "")
table.add_row("values", v_table)
alias_sum = self.alias_summary()
a_table = Table(box=box.SIMPLE, show_header=False)
a_table.add_column("Property")
a_table.add_column("Value")
a_table.add_row("no. aliases", str(len(self.aliases)))
a_table.add_row("combined size", format_size(alias_sum["size"]))
if full_details:
if self.aliases:
aliases = sorted(self.aliases.keys())
a_table.add_row(
"aliases", f"{aliases[0]} -> {self.aliases[aliases[0]]}"
)
for alias in aliases[1:]:
a_table.add_row("", f"{alias} -> {self.aliases[alias]}")
else:
a_table.add_row("aliases", "")
table.add_row("aliases", a_table)
if show_archive_info:
table.add_row("archives", self.archives)
return table
aliases: Dict[str, uuid.UUID]
pydantic-field
required
¶All available aliases within this context (and the value ids they refer to).
archives: ArchiveGroupInfo
pydantic-field
required
¶The archives registered in this context.
context_name: str
pydantic-field
¶The local alias for this context.
kiara_context: Kiara
property
readonly
¶kiara_id: UUID
pydantic-field
required
¶The (globally unique) id of the kiara context.
value_ids: List[uuid.UUID]
pydantic-field
required
¶The ids of all stored values in this context.
alias_summary(self)
¶Source code in kiara/models/context.py
def alias_summary(self) -> Dict[str, Any]:
sum_size = 0
types: Dict[str, int] = {}
internal_types: Dict[str, int] = {}
no_of_values = len(self.value_ids)
for alias, value_id in self.aliases.items():
value = self.kiara_context.data_registry.get_value(value=value_id)
sum_size = sum_size + value.value_size
if self.kiara_context.type_registry.is_internal_type(value.data_type_name):
if value.data_type_name not in internal_types.keys():
internal_types[value.data_type_name] = 1
else:
internal_types[value.data_type_name] += 1
else:
if value.data_type_name not in types.keys():
types[value.data_type_name] = 1
else:
types[value.data_type_name] += 1
types.setdefault(value.data_type_name, 0)
return {
"size": sum_size,
"no_values": no_of_values,
"types": types,
"internal_types": internal_types,
}
create_from_context(kiara, context_name=None)
classmethod
¶Source code in kiara/models/context.py
@classmethod
def create_from_context(cls, kiara: "Kiara", context_name: Union[str, None] = None):
value_ids = list(kiara.data_registry.retrieve_all_available_value_ids())
aliases = {
a.full_alias: a.value_id for a in kiara.alias_registry.aliases.values()
}
archives_info = ArchiveGroupInfo.create_from_context(kiara=kiara)
result = ContextSummary.construct(
kiara_id=kiara.id,
value_ids=value_ids,
aliases=aliases,
context_name=context_name,
archives=archives_info,
)
result._kiara = kiara
return result
create_from_context_config(config, context_name=None, runtime_config=None)
classmethod
¶Source code in kiara/models/context.py
@classmethod
def create_from_context_config(
cls,
config: KiaraContextConfig,
context_name: Union[str, None] = None,
runtime_config: Union[KiaraRuntimeConfig, None] = None,
):
from kiara.context import Kiara
kiara = Kiara(config=config, runtime_config=runtime_config)
return cls.create_from_context(kiara=kiara, context_name=context_name)
create_renderable(self, **config)
¶Source code in kiara/models/context.py
def create_renderable(self, **config: Any) -> RenderableType:
full_details = config.get("full_details", False)
show_value_ids = config.get("show_value_ids", False)
show_archive_info = config.get("show_archive_info", True)
table = Table(box=box.SIMPLE, show_header=False)
table.add_column("Property", style="i")
table.add_column("Value")
if self.context_name:
table.add_row("context name", self.context_name)
table.add_row("kiara_id", str(self.kiara_id))
size_on_disk = humanfriendly.format_size(self.archives.combined_size)
table.add_row("size on disk", size_on_disk)
value_sum = self.value_summary()
v_table = Table(box=box.SIMPLE, show_header=False)
v_table.add_column("Property")
v_table.add_column("Value")
v_table.add_row("no. values", str(value_sum["no_values"]))
v_table.add_row("combined size", format_size(value_sum["size"]))
if full_details and show_value_ids:
if self.value_ids:
value_ids = sorted((str(v) for v in self.value_ids))
v_table.add_row("value_ids", value_ids[0])
for v_id in value_ids[1:]:
v_table.add_row("", v_id)
else:
v_table.add_row("value_ids", "")
table.add_row("values", v_table)
alias_sum = self.alias_summary()
a_table = Table(box=box.SIMPLE, show_header=False)
a_table.add_column("Property")
a_table.add_column("Value")
a_table.add_row("no. aliases", str(len(self.aliases)))
a_table.add_row("combined size", format_size(alias_sum["size"]))
if full_details:
if self.aliases:
aliases = sorted(self.aliases.keys())
a_table.add_row(
"aliases", f"{aliases[0]} -> {self.aliases[aliases[0]]}"
)
for alias in aliases[1:]:
a_table.add_row("", f"{alias} -> {self.aliases[alias]}")
else:
a_table.add_row("aliases", "")
table.add_row("aliases", a_table)
if show_archive_info:
table.add_row("archives", self.archives)
return table
value_summary(self)
¶Source code in kiara/models/context.py
def value_summary(self) -> Dict[str, Any]:
sum_size = 0
types: Dict[str, int] = {}
internal_types: Dict[str, int] = {}
no_of_values = len(self.value_ids)
for value_id in self.value_ids:
value = self.kiara_context.data_registry.get_value(value=value_id)
sum_size = sum_size + value.value_size
if self.kiara_context.type_registry.is_internal_type(value.data_type_name):
if value.data_type_name not in internal_types.keys():
internal_types[value.data_type_name] = 1
else:
internal_types[value.data_type_name] += 1
else:
if value.data_type_name not in types.keys():
types[value.data_type_name] = 1
else:
types[value.data_type_name] += 1
types.setdefault(value.data_type_name, 0)
return {
"size": sum_size,
"no_values": no_of_values,
"types": types,
"internal_types": internal_types,
}
data_types
¶
This module contains the metadata (and other) models that are used in the kiara_plugin.core_types package.
Those models are convenience wrappers that make it easier for kiara to find, create, manage and version metadata -- but also other type of models -- that is attached to data, as well as kiara modules.
Metadata models must be a sub-class of [kiara.metadata.MetadataModel][]. Other models usually sub-class a pydantic BaseModel or implement custom base classes.
Classes¶
DictModel (BaseModel, Mapping, Generic)
pydantic-model
¶A dict implentation that contains (optional) schema information of the dicts items.
Source code in kiara/models/data_types.py
class DictModel(BaseModel, Mapping):
"""A dict implentation that contains (optional) schema information of the dicts items."""
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
dict_data: Dict[str, Any] = Field(description="The data.")
data_schema: Dict[str, Any] = Field(description="The schema.")
python_class: PythonClass = Field(
description="The python class of which model instances are created. This is mostly meant as a hint for client applications."
)
_size_cache: int = PrivateAttr(default=None)
_hash_cache: int = PrivateAttr(default=None)
_data_hash: int = PrivateAttr(default=None)
_schema_hash: int = PrivateAttr(default=None)
_value_hash: int = PrivateAttr(default=None)
@property
def size(self):
if self._size_cache is not None:
return self._size_cache
self._size_cache = len(self.dict_data) + len(self.data_schema)
return self._size_cache
@property
def data_hash(self) -> int:
if self._data_hash is not None:
return self._data_hash
self._data_hash = compute_cid(self.dict_data)
return self._data_hash
@property
def schema_hash(self) -> int:
if self._schema_hash is not None:
return self._schema_hash
self._schema_hash = compute_cid(self.data_schema)
return self._schema_hash
@property
def value_hash(self) -> int:
if self._value_hash is not None:
return self._value_hash
obj = {"data": self.data_hash, "data_schema": self.schema_hash}
self._value_hash = compute_cid(obj)
return self._value_hash
def __getitem__(self, item):
return self.dict_data.__getitem__(item)
def __iter__(self):
return self.dict_data.__iter__()
def __len__(self):
return self.dict_data.__len__()
data_hash: int
property
readonly
¶data_schema: Dict[str, Any]
pydantic-field
required
¶The schema.
dict_data: Dict[str, Any]
pydantic-field
required
¶The data.
python_class: PythonClass
pydantic-field
required
¶The python class of which model instances are created. This is mostly meant as a hint for client applications.
schema_hash: int
property
readonly
¶size
property
readonly
¶value_hash: int
property
readonly
¶
Config
¶Source code in kiara/models/data_types.py
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
json_loads
¶json_dumps(v, *, default=None, **args)
¶Source code in kiara/models/data_types.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
documentation
¶
Classes¶
AuthorModel (BaseModel)
pydantic-model
¶Details about an author of a resource.
Source code in kiara/models/documentation.py
class AuthorModel(BaseModel):
"""Details about an author of a resource."""
class Config:
title = "Author"
name: str = Field(description="The full name of the author.")
email: Union[EmailStr, None] = Field(
description="The email address of the author", default=None
)
AuthorsMetadataModel (KiaraModel)
pydantic-model
¶Information about all authors of a resource.
Source code in kiara/models/documentation.py
class AuthorsMetadataModel(KiaraModel):
"""Information about all authors of a resource."""
_kiara_model_id = "metadata.authors"
class Config:
extra = Extra.ignore
title = "Authors"
_metadata_key = "origin"
@classmethod
def from_class(cls, item_cls: Type):
data = get_metadata_for_python_module_or_class(item_cls) # type: ignore
merged = merge_dicts(*data)
return cls.parse_obj(merged)
authors: List[AuthorModel] = Field(
description="The authors/creators of this item.", default_factory=list
)
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Name")
table.add_column("Email", style="i")
for author in reversed(self.authors):
if author.email:
authors: Tuple[str, Union[str, EmailStr]] = (author.name, author.email)
else:
authors = (author.name, "")
table.add_row(*authors)
return table
authors: List[kiara.models.documentation.AuthorModel]
pydantic-field
¶The authors/creators of this item.
Config
¶create_renderable(self, **config)
¶Source code in kiara/models/documentation.py
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Name")
table.add_column("Email", style="i")
for author in reversed(self.authors):
if author.email:
authors: Tuple[str, Union[str, EmailStr]] = (author.name, author.email)
else:
authors = (author.name, "")
table.add_row(*authors)
return table
from_class(item_cls)
classmethod
¶Source code in kiara/models/documentation.py
@classmethod
def from_class(cls, item_cls: Type):
data = get_metadata_for_python_module_or_class(item_cls) # type: ignore
merged = merge_dicts(*data)
return cls.parse_obj(merged)
ContextMetadataModel (KiaraModel)
pydantic-model
¶Information about the context of a resource.
Source code in kiara/models/documentation.py
class ContextMetadataModel(KiaraModel):
"""Information about the context of a resource."""
_kiara_model_id = "metadata.context"
class Config:
extra = Extra.ignore
title = "Context"
@classmethod
def from_class(cls, item_cls: Type):
data = get_metadata_for_python_module_or_class(item_cls) # type: ignore
merged = merge_dicts(*data)
return cls.parse_obj(merged)
_metadata_key = "properties"
references: Dict[str, LinkModel] = Field(
description="References for the item.", default_factory=dict
)
tags: List[str] = Field(
description="A list of tags for the item.", default_factory=list
)
labels: Dict[str, str] = Field(
description="A list of labels for the item.", default_factory=list
)
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Key", style="i")
table.add_column("Value")
if self.tags:
table.add_row("Tags", ", ".join(self.tags))
if self.labels:
labels = []
for k, v in self.labels.items():
labels.append(f"[i]{k}[/i]: {v}")
table.add_row("Labels", "\n".join(labels))
if self.references:
references = []
for _k, _v in self.references.items():
link = f"[link={_v.url}]{_v.url}[/link]"
references.append(f"[i]{_k}[/i]: {link}")
table.add_row("References", "\n".join(references))
return table
def add_reference(
self,
ref_type: str,
url: str,
desc: Union[str, None] = None,
force: bool = False,
):
if ref_type in self.references.keys() and not force:
raise Exception(f"Reference of type '{ref_type}' already present.")
link = LinkModel(url=url, desc=desc)
self.references[ref_type] = link
def get_url_for_reference(self, ref: str) -> Union[str, None]:
link = self.references.get(ref, None)
if not link:
return None
return link.url
labels: Dict[str, str]
pydantic-field
¶A list of labels for the item.
references: Dict[str, kiara.models.documentation.LinkModel]
pydantic-field
¶References for the item.
tags: List[str]
pydantic-field
¶A list of tags for the item.
Config
¶add_reference(self, ref_type, url, desc=None, force=False)
¶Source code in kiara/models/documentation.py
def add_reference(
self,
ref_type: str,
url: str,
desc: Union[str, None] = None,
force: bool = False,
):
if ref_type in self.references.keys() and not force:
raise Exception(f"Reference of type '{ref_type}' already present.")
link = LinkModel(url=url, desc=desc)
self.references[ref_type] = link
create_renderable(self, **config)
¶Source code in kiara/models/documentation.py
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Key", style="i")
table.add_column("Value")
if self.tags:
table.add_row("Tags", ", ".join(self.tags))
if self.labels:
labels = []
for k, v in self.labels.items():
labels.append(f"[i]{k}[/i]: {v}")
table.add_row("Labels", "\n".join(labels))
if self.references:
references = []
for _k, _v in self.references.items():
link = f"[link={_v.url}]{_v.url}[/link]"
references.append(f"[i]{_k}[/i]: {link}")
table.add_row("References", "\n".join(references))
return table
from_class(item_cls)
classmethod
¶Source code in kiara/models/documentation.py
@classmethod
def from_class(cls, item_cls: Type):
data = get_metadata_for_python_module_or_class(item_cls) # type: ignore
merged = merge_dicts(*data)
return cls.parse_obj(merged)
get_url_for_reference(self, ref)
¶Source code in kiara/models/documentation.py
def get_url_for_reference(self, ref: str) -> Union[str, None]:
link = self.references.get(ref, None)
if not link:
return None
return link.url
DocumentationMetadataModel (KiaraModel)
pydantic-model
¶Documentation about a resource.
Source code in kiara/models/documentation.py
class DocumentationMetadataModel(KiaraModel):
"""Documentation about a resource."""
class Config:
title = "Documentation"
_kiara_model_id = "metadata.documentation"
_metadata_key = "documentation"
@classmethod
def from_class_doc(cls, item_cls: Type):
doc = item_cls.__doc__
if not doc:
doc = DEFAULT_NO_DESC_VALUE
doc = inspect.cleandoc(doc)
return cls.from_string(doc)
@classmethod
def from_function(cls, func: Callable):
doc = func.__doc__
if not doc:
doc = DEFAULT_NO_DESC_VALUE
doc = inspect.cleandoc(doc)
return cls.from_string(doc)
@classmethod
def from_string(cls, doc: Union[str, None]):
if not doc:
doc = DEFAULT_NO_DESC_VALUE
if "\n" in doc:
desc, doc = doc.split("\n", maxsplit=1)
else:
desc = doc
doc = None
if doc:
doc = doc.strip()
return cls(description=desc.strip(), doc=doc)
@classmethod
def from_dict(cls, data: Mapping):
doc = data.get("doc", None)
desc = data.get("description", None)
if desc is None:
desc = data.get("desc", None)
if not doc and not desc:
return cls.from_string(DEFAULT_NO_DESC_VALUE)
elif doc and not desc:
return cls.from_string(doc)
elif desc and not doc:
return cls.from_string(desc)
else:
return cls(description=desc, doc=doc)
@classmethod
def create(cls, item: Any = None):
if not item:
return cls.from_string(DEFAULT_NO_DESC_VALUE)
elif isinstance(item, DocumentationMetadataModel):
return item
elif isinstance(item, Mapping):
return cls.from_dict(item)
if isinstance(item, type):
return cls.from_class_doc(item)
elif isinstance(item, str):
return cls.from_string(item)
else:
raise TypeError(f"Can't create documentation from type '{type(item)}'.")
description: str = Field(
description="Short description of the item.", default=DEFAULT_NO_DESC_VALUE
)
doc: Union[str, None] = Field(
description="Detailed documentation of the item (in markdown).", default=None
)
@property
def is_set(self) -> bool:
if self.description and self.description != DEFAULT_NO_DESC_VALUE:
return True
else:
return False
def _retrieve_data_to_hash(self) -> Any:
return self.full_doc
@property
def full_doc(self):
if self.doc:
return f"{self.description}\n\n{self.doc}"
else:
return self.description
def create_renderable(self, **config: Any) -> RenderableType:
return Markdown(self.full_doc)
description: str
pydantic-field
¶Short description of the item.
doc: str
pydantic-field
¶Detailed documentation of the item (in markdown).
full_doc
property
readonly
¶is_set: bool
property
readonly
¶
Config
¶Source code in kiara/models/documentation.py
class Config:
title = "Documentation"
create(item=None)
classmethod
¶Source code in kiara/models/documentation.py
@classmethod
def create(cls, item: Any = None):
if not item:
return cls.from_string(DEFAULT_NO_DESC_VALUE)
elif isinstance(item, DocumentationMetadataModel):
return item
elif isinstance(item, Mapping):
return cls.from_dict(item)
if isinstance(item, type):
return cls.from_class_doc(item)
elif isinstance(item, str):
return cls.from_string(item)
else:
raise TypeError(f"Can't create documentation from type '{type(item)}'.")
create_renderable(self, **config)
¶Source code in kiara/models/documentation.py
def create_renderable(self, **config: Any) -> RenderableType:
return Markdown(self.full_doc)
from_class_doc(item_cls)
classmethod
¶Source code in kiara/models/documentation.py
@classmethod
def from_class_doc(cls, item_cls: Type):
doc = item_cls.__doc__
if not doc:
doc = DEFAULT_NO_DESC_VALUE
doc = inspect.cleandoc(doc)
return cls.from_string(doc)
from_dict(data)
classmethod
¶Source code in kiara/models/documentation.py
@classmethod
def from_dict(cls, data: Mapping):
doc = data.get("doc", None)
desc = data.get("description", None)
if desc is None:
desc = data.get("desc", None)
if not doc and not desc:
return cls.from_string(DEFAULT_NO_DESC_VALUE)
elif doc and not desc:
return cls.from_string(doc)
elif desc and not doc:
return cls.from_string(desc)
else:
return cls(description=desc, doc=doc)
from_function(func)
classmethod
¶Source code in kiara/models/documentation.py
@classmethod
def from_function(cls, func: Callable):
doc = func.__doc__
if not doc:
doc = DEFAULT_NO_DESC_VALUE
doc = inspect.cleandoc(doc)
return cls.from_string(doc)
from_string(doc)
classmethod
¶Source code in kiara/models/documentation.py
@classmethod
def from_string(cls, doc: Union[str, None]):
if not doc:
doc = DEFAULT_NO_DESC_VALUE
if "\n" in doc:
desc, doc = doc.split("\n", maxsplit=1)
else:
desc = doc
doc = None
if doc:
doc = doc.strip()
return cls(description=desc.strip(), doc=doc)
LinkModel (BaseModel)
pydantic-model
¶A description and url for a reference of any kind.
Source code in kiara/models/documentation.py
class LinkModel(BaseModel):
"""A description and url for a reference of any kind."""
class Config:
title = "Link"
url: AnyUrl = Field(description="The url.")
desc: Union[str, None] = Field(
description="A short description of the link content.",
default=DEFAULT_NO_DESC_VALUE,
)
events
special
¶
Classes¶
KiaraEvent (BaseModel)
pydantic-model
¶Source code in kiara/models/events/__init__.py
class KiaraEvent(BaseModel):
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
def get_event_type(self) -> str:
if hasattr(self, "event_type"):
return self.event_type # type: ignore
name = camel_case_to_snake_case(self.__class__.__name__)
return name
Config
¶Source code in kiara/models/events/__init__.py
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
json_loads
¶json_dumps(v, *, default=None, **args)
¶Source code in kiara/models/events/__init__.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
get_event_type(self)
¶Source code in kiara/models/events/__init__.py
def get_event_type(self) -> str:
if hasattr(self, "event_type"):
return self.event_type # type: ignore
name = camel_case_to_snake_case(self.__class__.__name__)
return name
RegistryEvent (KiaraEvent)
pydantic-model
¶Modules¶
alias_registry
¶
AliasArchiveAddedEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/alias_registry.py
class AliasArchiveAddedEvent(RegistryEvent):
event_type: Literal["alias_archive_added"] = "alias_archive_added"
alias_archive_id: uuid.UUID = Field(
description="The unique id of this data archive."
)
alias_archive_alias: str = Field(
description="The alias this data archive was added as."
)
is_store: bool = Field(
description="Whether this archive supports write operations (aka implements the 'DataStore' interface)."
)
is_default_store: bool = Field(
description="Whether this store acts as default store."
)
alias_archive_alias: str
pydantic-field
required
¶The alias this data archive was added as.
alias_archive_id: UUID
pydantic-field
required
¶The unique id of this data archive.
event_type: Literal['alias_archive_added']
pydantic-field
¶is_default_store: bool
pydantic-field
required
¶Whether this store acts as default store.
is_store: bool
pydantic-field
required
¶Whether this archive supports write operations (aka implements the 'DataStore' interface).
AliasPreStoreEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/alias_registry.py
class AliasPreStoreEvent(RegistryEvent):
event_type: Literal["alias_pre_store"] = "alias_pre_store"
aliases: Iterable[str] = Field(description="The alias.")
AliasStoredEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/alias_registry.py
class AliasStoredEvent(RegistryEvent):
event_type: Literal["alias_stored"] = "alias_stored"
alias: str = Field(description="The alias.")
data_registry
¶
DataArchiveAddedEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/data_registry.py
class DataArchiveAddedEvent(RegistryEvent):
event_type: Literal["data_archive_added"] = "data_archive_added"
data_archive_id: uuid.UUID = Field(
description="The unique id of this data archive."
)
data_archive_alias: str = Field(
description="The alias this data archive was added as."
)
is_store: bool = Field(
description="Whether this archive supports write operations (aka implements the 'DataStore' interface)."
)
is_default_store: bool = Field(
description="Whether this store acts as default store."
)
data_archive_alias: str
pydantic-field
required
¶The alias this data archive was added as.
data_archive_id: UUID
pydantic-field
required
¶The unique id of this data archive.
event_type: Literal['data_archive_added']
pydantic-field
¶is_default_store: bool
pydantic-field
required
¶Whether this store acts as default store.
is_store: bool
pydantic-field
required
¶Whether this archive supports write operations (aka implements the 'DataStore' interface).
ValueCreatedEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/data_registry.py
class ValueCreatedEvent(RegistryEvent):
event_type: Literal["value_created"] = "value_created"
value: Value = Field(description="The value metadata.")
ValuePreStoreEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/data_registry.py
class ValuePreStoreEvent(RegistryEvent):
event_type: Literal["value_pre_store"] = "value_pre_store"
value: Value = Field(description="The value metadata.")
ValueRegisteredEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/data_registry.py
class ValueRegisteredEvent(RegistryEvent):
event_type: Literal["value_registered"] = "value_registered"
value: Value = Field(description="The value metadata.")
ValueStoredEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/data_registry.py
class ValueStoredEvent(RegistryEvent):
event_type: Literal["value_stored"] = "value_stored"
value: Value = Field(description="The value metadata.")
destiny_registry
¶
DestinyArchiveAddedEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/destiny_registry.py
class DestinyArchiveAddedEvent(RegistryEvent):
event_type: Literal["destiny_archive_added"] = "destiny_archive_added"
destiny_archive_id: uuid.UUID = Field(
description="The unique id of this destiny archive."
)
destiny_archive_alias: str = Field(
description="The alias this destiny archive was added as."
)
is_store: bool = Field(
description="Whether this archive supports write operations (aka implements the 'DestinyStore' interface)."
)
is_default_store: bool = Field(
description="Whether this store acts as default store."
)
destiny_archive_alias: str
pydantic-field
required
¶The alias this destiny archive was added as.
destiny_archive_id: UUID
pydantic-field
required
¶The unique id of this destiny archive.
event_type: Literal['destiny_archive_added']
pydantic-field
¶is_default_store: bool
pydantic-field
required
¶Whether this store acts as default store.
is_store: bool
pydantic-field
required
¶Whether this archive supports write operations (aka implements the 'DestinyStore' interface).
job_registry
¶
JobArchiveAddedEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/job_registry.py
class JobArchiveAddedEvent(RegistryEvent):
event_type: Literal["job_archive_added"] = "job_archive_added"
job_archive_id: uuid.UUID = Field(description="The unique id of this job archive.")
job_archive_alias: str = Field(
description="The alias this job archive was added as."
)
is_store: bool = Field(
description="Whether this archive supports write operations (aka implements the 'JobStore' interface)."
)
is_default_store: bool = Field(
description="Whether this store acts as default store."
)
event_type: Literal['job_archive_added']
pydantic-field
¶is_default_store: bool
pydantic-field
required
¶Whether this store acts as default store.
is_store: bool
pydantic-field
required
¶Whether this archive supports write operations (aka implements the 'JobStore' interface).
job_archive_alias: str
pydantic-field
required
¶The alias this job archive was added as.
job_archive_id: UUID
pydantic-field
required
¶The unique id of this job archive.
JobRecordPreStoreEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/job_registry.py
class JobRecordPreStoreEvent(RegistryEvent):
event_type: Literal["job_record_pre_store"] = "job_record_pre_store"
job_record: JobRecord = Field(description="The job record.")
JobRecordStoredEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/job_registry.py
class JobRecordStoredEvent(RegistryEvent):
event_type: Literal["job_record_stored"] = "job_record_stored"
job_record: JobRecord = Field(description="The job record.")
pipeline
¶
ChangedValue (BaseModel)
pydantic-model
¶
PipelineDetails (BaseModel)
pydantic-model
¶Source code in kiara/models/events/pipeline.py
class PipelineDetails(BaseModel):
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
kiara_id: uuid.UUID = Field(description="The id of the kiara context.")
pipeline_id: uuid.UUID = Field(description="The id of the pipeline.")
pipeline_status: StepStatus = Field(
description="The current status of this pipeline."
)
invalid_details: Dict[str, str] = Field(
description="Details about fields that are invalid (if status < 'INPUTS_READY'.",
default_factory=dict,
)
pipeline_inputs: Dict[str, uuid.UUID] = Field(
description="The current pipeline inputs."
)
pipeline_outputs: Dict[str, uuid.UUID] = Field(
description="The current pipeline outputs."
)
step_states: Dict[str, StepDetails] = Field(
description="The state of each step within this pipeline."
)
def get_steps_by_processing_stage(self) -> MutableMapping[int, List[StepDetails]]:
result: MutableMapping[int, List[StepDetails]] = SortedDict()
for step_details in self.step_states.values():
result.setdefault(step_details.processing_stage, []).append(step_details)
return result
invalid_details: Dict[str, str]
pydantic-field
¶Details about fields that are invalid (if status < 'INPUTS_READY'.
kiara_id: UUID
pydantic-field
required
¶The id of the kiara context.
pipeline_id: UUID
pydantic-field
required
¶The id of the pipeline.
pipeline_inputs: Dict[str, uuid.UUID]
pydantic-field
required
¶The current pipeline inputs.
pipeline_outputs: Dict[str, uuid.UUID]
pydantic-field
required
¶The current pipeline outputs.
pipeline_status: StepStatus
pydantic-field
required
¶The current status of this pipeline.
step_states: Dict[str, kiara.models.events.pipeline.StepDetails]
pydantic-field
required
¶The state of each step within this pipeline.
Config
¶Source code in kiara/models/events/pipeline.py
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
json_loads
¶json_dumps(v, *, default=None, **args)
¶Source code in kiara/models/events/pipeline.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
get_steps_by_processing_stage(self)
¶Source code in kiara/models/events/pipeline.py
def get_steps_by_processing_stage(self) -> MutableMapping[int, List[StepDetails]]:
result: MutableMapping[int, List[StepDetails]] = SortedDict()
for step_details in self.step_states.values():
result.setdefault(step_details.processing_stage, []).append(step_details)
return result
PipelineEvent (KiaraEvent)
pydantic-model
¶Source code in kiara/models/events/pipeline.py
class PipelineEvent(KiaraEvent):
@classmethod
def create_event(
cls,
pipeline: "Pipeline",
changed: Mapping[str, Mapping[str, Mapping[str, ChangedValue]]],
) -> Union["PipelineEvent", None]:
pipeline_inputs = changed.get("__pipeline__", {}).get("inputs", {})
pipeline_outputs = changed.get("__pipeline__", {}).get("outputs", {})
step_inputs = {}
step_outputs = {}
invalidated_steps: Set[str] = set()
for step_id, change_details in changed.items():
if step_id == "__pipeline__":
continue
inputs = change_details.get("inputs", None)
if inputs:
invalidated_steps.add(step_id)
step_inputs[step_id] = inputs
outputs = change_details.get("outputs", None)
if outputs:
invalidated_steps.add(step_id)
step_outputs[step_id] = outputs
if (
not pipeline_inputs
and not pipeline_outputs
and not step_inputs
and not step_outputs
and not invalidated_steps
):
return None
event = PipelineEvent(
kiara_id=pipeline.kiara_id,
pipeline_id=pipeline.pipeline_id,
pipeline_inputs_changed=pipeline_inputs,
pipeline_outputs_changed=pipeline_outputs,
step_inputs_changed=step_inputs,
step_outputs_changed=step_outputs,
changed_steps=sorted(invalidated_steps),
)
return event
class Config:
allow_mutation = False
kiara_id: uuid.UUID = Field(
description="The id of the kiara context that created the pipeline."
)
pipeline_id: uuid.UUID = Field(description="The pipeline id.")
pipeline_inputs_changed: Dict[str, ChangedValue] = Field(
description="Details about changed pipeline input values.", default_factory=dict
)
pipeline_outputs_changed: Dict[str, ChangedValue] = Field(
description="Details about changed pipeline output values.",
default_factory=dict,
)
step_inputs_changed: Dict[str, Mapping[str, ChangedValue]] = Field(
description="Details about changed step input values.", default_factory=dict
)
step_outputs_changed: Dict[str, Mapping[str, ChangedValue]] = Field(
description="Details about changed step output values.", default_factory=dict
)
changed_steps: List[str] = Field(
description="A list of all step ids that have newly invalidated outputs."
)
def __repr__(self):
return f"{self.__class__.__name__}(pipeline_id={self.pipeline_id}, invalidated_steps={', '.join(self.changed_steps)})"
def __str__(self):
return self.__repr__()
changed_steps: List[str]
pydantic-field
required
¶A list of all step ids that have newly invalidated outputs.
kiara_id: UUID
pydantic-field
required
¶The id of the kiara context that created the pipeline.
pipeline_id: UUID
pydantic-field
required
¶The pipeline id.
pipeline_inputs_changed: Dict[str, kiara.models.events.pipeline.ChangedValue]
pydantic-field
¶Details about changed pipeline input values.
pipeline_outputs_changed: Dict[str, kiara.models.events.pipeline.ChangedValue]
pydantic-field
¶Details about changed pipeline output values.
step_inputs_changed: Dict[str, Mapping[str, kiara.models.events.pipeline.ChangedValue]]
pydantic-field
¶Details about changed step input values.
step_outputs_changed: Dict[str, Mapping[str, kiara.models.events.pipeline.ChangedValue]]
pydantic-field
¶Details about changed step output values.
Config
¶Source code in kiara/models/events/pipeline.py
class Config:
allow_mutation = False
create_event(pipeline, changed)
classmethod
¶Source code in kiara/models/events/pipeline.py
@classmethod
def create_event(
cls,
pipeline: "Pipeline",
changed: Mapping[str, Mapping[str, Mapping[str, ChangedValue]]],
) -> Union["PipelineEvent", None]:
pipeline_inputs = changed.get("__pipeline__", {}).get("inputs", {})
pipeline_outputs = changed.get("__pipeline__", {}).get("outputs", {})
step_inputs = {}
step_outputs = {}
invalidated_steps: Set[str] = set()
for step_id, change_details in changed.items():
if step_id == "__pipeline__":
continue
inputs = change_details.get("inputs", None)
if inputs:
invalidated_steps.add(step_id)
step_inputs[step_id] = inputs
outputs = change_details.get("outputs", None)
if outputs:
invalidated_steps.add(step_id)
step_outputs[step_id] = outputs
if (
not pipeline_inputs
and not pipeline_outputs
and not step_inputs
and not step_outputs
and not invalidated_steps
):
return None
event = PipelineEvent(
kiara_id=pipeline.kiara_id,
pipeline_id=pipeline.pipeline_id,
pipeline_inputs_changed=pipeline_inputs,
pipeline_outputs_changed=pipeline_outputs,
step_inputs_changed=step_inputs,
step_outputs_changed=step_outputs,
changed_steps=sorted(invalidated_steps),
)
return event
StepDetails (BaseModel)
pydantic-model
¶Source code in kiara/models/events/pipeline.py
class StepDetails(BaseModel):
kiara_id: uuid.UUID = Field(description="The id of the kiara context.")
pipeline_id: uuid.UUID = Field(description="The id of the pipeline.")
step: PipelineStep = Field(description="The pipeline step details.")
step_id: str = Field(description="The id of the step.")
processing_stage: int = Field(
description="The execution stage where this step is executed."
)
status: StepStatus = Field(description="The current status of this step.")
invalid_details: Dict[str, str] = Field(
description="Details about fields that are invalid (if status < 'INPUTS_READY'.",
default_factory=dict,
)
inputs: Dict[str, uuid.UUID] = Field(description="The current inputs of this step.")
outputs: Dict[str, uuid.UUID] = Field(
description="The current outputs of this step."
)
@validator("inputs")
def replace_none_values_inputs(cls, value):
result = {}
for k, v in value.items():
if v is None:
v = NONE_VALUE_ID
result[k] = v
return result
@validator("outputs")
def replace_none_values_outputs(cls, value):
result = {}
for k, v in value.items():
if v is None:
v = NOT_SET_VALUE_ID
result[k] = v
return result
def _retrieve_data_to_hash(self) -> Any:
return f"{self.kiara_id}.{self.pipeline_id}.{self.step_id}"
def _retrieve_id(self) -> str:
return f"{self.kiara_id}.{self.pipeline_id}.{self.step_id}"
inputs: Dict[str, uuid.UUID]
pydantic-field
required
¶The current inputs of this step.
invalid_details: Dict[str, str]
pydantic-field
¶Details about fields that are invalid (if status < 'INPUTS_READY'.
kiara_id: UUID
pydantic-field
required
¶The id of the kiara context.
outputs: Dict[str, uuid.UUID]
pydantic-field
required
¶The current outputs of this step.
pipeline_id: UUID
pydantic-field
required
¶The id of the pipeline.
processing_stage: int
pydantic-field
required
¶The execution stage where this step is executed.
status: StepStatus
pydantic-field
required
¶The current status of this step.
step: PipelineStep
pydantic-field
required
¶The pipeline step details.
step_id: str
pydantic-field
required
¶The id of the step.
replace_none_values_inputs(value)
classmethod
¶Source code in kiara/models/events/pipeline.py
@validator("inputs")
def replace_none_values_inputs(cls, value):
result = {}
for k, v in value.items():
if v is None:
v = NONE_VALUE_ID
result[k] = v
return result
replace_none_values_outputs(value)
classmethod
¶Source code in kiara/models/events/pipeline.py
@validator("outputs")
def replace_none_values_outputs(cls, value):
result = {}
for k, v in value.items():
if v is None:
v = NOT_SET_VALUE_ID
result[k] = v
return result
workflow_registry
¶
WorkflowArchiveAddedEvent (RegistryEvent)
pydantic-model
¶Source code in kiara/models/events/workflow_registry.py
class WorkflowArchiveAddedEvent(RegistryEvent):
event_type: Literal["workflow_archive_added"] = "workflow_archive_added"
workflow_archive_id: uuid.UUID = Field(
description="The unique id of this data archive."
)
workflow_archive_alias: str = Field(
description="The alias this workflow archive was added as."
)
is_store: bool = Field(
description="Whether this archive supports write operations (aka implements the 'WorkflowStore' interface)."
)
is_default_store: bool = Field(
description="Whether this store acts as default store."
)
event_type: Literal['workflow_archive_added']
pydantic-field
¶is_default_store: bool
pydantic-field
required
¶Whether this store acts as default store.
is_store: bool
pydantic-field
required
¶Whether this archive supports write operations (aka implements the 'WorkflowStore' interface).
workflow_archive_alias: str
pydantic-field
required
¶The alias this workflow archive was added as.
workflow_archive_id: UUID
pydantic-field
required
¶The unique id of this data archive.
filesystem
¶
FILE_BUNDLE_IMPORT_AVAILABLE_COLUMNS
¶logger
¶Classes¶
FileBundle (KiaraModel)
pydantic-model
¶Describes properties for the 'file_bundle' value type.
Source code in kiara/models/filesystem.py
class FileBundle(KiaraModel):
"""Describes properties for the 'file_bundle' value type."""
_kiara_model_id = "instance.data.file_bundle"
@classmethod
def create_tmp_dir(self) -> Path:
"""Utility method to create a temp folder that gets deleted when kiara exits."""
temp_f = tempfile.mkdtemp()
def cleanup():
shutil.rmtree(temp_f, ignore_errors=True)
atexit.register(cleanup)
return Path(temp_f)
@classmethod
def import_folder(
cls,
source: str,
bundle_name: Union[str, None] = None,
import_config: Union[None, Mapping[str, Any], FolderImportConfig] = None,
# import_time: Optional[datetime.datetime] = None,
) -> "FileBundle":
if not source:
raise ValueError("No source path provided.")
if not os.path.exists(os.path.realpath(source)):
raise ValueError(f"Path does not exist: {source}")
if not os.path.isdir(os.path.realpath(source)):
raise ValueError(f"Path is not a folder: {source}")
if source.endswith(os.path.sep):
source = source[0:-1]
abs_path = os.path.abspath(source)
if import_config is None:
_import_config = FolderImportConfig()
elif isinstance(import_config, Mapping):
_import_config = FolderImportConfig(**import_config)
elif isinstance(import_config, FolderImportConfig):
_import_config = import_config
else:
raise TypeError(
f"Invalid type for folder import config: {type(import_config)}."
)
included_files: Dict[str, FileModel] = {}
exclude_dirs = _import_config.exclude_dirs
invalid_extensions = _import_config.exclude_files
valid_extensions = _import_config.include_files
# if import_time:
# bundle_import_time = import_time
# else:
# bundle_import_time = datetime.datetime.now() # TODO: timezone
sum_size = 0
def include_file(filename: str) -> bool:
if invalid_extensions and any(
filename.endswith(ext) for ext in invalid_extensions
):
return False
if not valid_extensions:
return True
else:
return any(filename.endswith(ext) for ext in valid_extensions)
for root, dirnames, filenames in os.walk(abs_path, topdown=True):
if exclude_dirs:
dirnames[:] = [d for d in dirnames if d not in exclude_dirs]
for filename in [
f
for f in filenames
if os.path.isfile(os.path.join(root, f)) and include_file(f)
]:
full_path = os.path.join(root, filename)
rel_path = os.path.relpath(full_path, abs_path)
file_model = FileModel.load_file(full_path)
sum_size = sum_size + file_model.size
included_files[rel_path] = file_model
if bundle_name is None:
bundle_name = os.path.basename(source)
bundle = FileBundle.create_from_file_models(
files=included_files,
path=abs_path,
bundle_name=bundle_name,
sum_size=sum_size,
)
return bundle
@classmethod
def create_from_file_models(
cls,
files: Mapping[str, FileModel],
bundle_name: str,
path: Union[str, None] = None,
sum_size: Union[int, None] = None,
# import_time: Optional[datetime.datetime] = None,
) -> "FileBundle":
# if import_time:
# bundle_import_time = import_time
# else:
# bundle_import_time = datetime.datetime.now() # TODO: timezone
result: Dict[str, Any] = {}
result["included_files"] = files
# result["import_time"] = datetime.datetime.now().isoformat()
result["number_of_files"] = len(files)
result["bundle_name"] = bundle_name
# result["import_time"] = bundle_import_time
if sum_size is None:
sum_size = 0
for f in files.values():
sum_size = sum_size + f.size
result["size"] = sum_size
bundle = FileBundle(**result)
bundle._path = path
return bundle
_file_bundle_hash: Union[int, None] = PrivateAttr(default=None)
bundle_name: str = Field(description="The name of this bundle.")
# import_time: datetime.datetime = Field(
# description="The time when the file bundle was imported."
# )
number_of_files: int = Field(
description="How many files are included in this bundle."
)
included_files: Dict[str, FileModel] = Field(
description="A map of all the included files, incl. their properties. Uses the relative path of each file as key."
)
size: int = Field(description="The size of all files in this folder, combined.")
_path: Union[str, None] = PrivateAttr(default=None)
@property
def path(self) -> str:
if self._path is None:
# TODO: better explanation, offer remedy like copying into temp folder
raise Exception(
"File bundle path not set, it appears this bundle is comprised of symlinks only."
)
return self._path
def _retrieve_id(self) -> str:
return str(self.file_bundle_hash)
# @property
# def model_data_hash(self) -> int:
# return self.file_bundle_hash
def _retrieve_data_to_hash(self) -> Any:
return {
"bundle_name": self.bundle_name,
"included_files": {
k: v.instance_cid for k, v in self.included_files.items()
},
}
def get_relative_path(self, file: FileModel):
return os.path.relpath(file.path, self.path)
def read_text_file_contents(self, ignore_errors: bool = False) -> Mapping[str, str]:
content_dict: Dict[str, str] = {}
def read_file(rel_path: str, full_path: str):
with open(full_path, encoding="utf-8") as f:
try:
content = f.read()
content_dict[rel_path] = content # type: ignore
except Exception as e:
if ignore_errors:
log_message(f"Can't read file: {e}")
logger.warning("ignore.file", path=full_path, reason=str(e))
else:
raise Exception(f"Can't read file (as text) '{full_path}: {e}")
# TODO: common ignore files and folders
for rel_path, f in self.included_files.items():
if f._path:
path = f._path
else:
path = self.get_relative_path(f)
read_file(rel_path=rel_path, full_path=path)
return content_dict
@property
def file_bundle_hash(self) -> int:
# TODO: use sha256?
if self._file_bundle_hash is not None:
return self._file_bundle_hash
obj = {k: v.file_hash for k, v in self.included_files.items()}
h = DeepHash(obj, hasher=KIARA_HASH_FUNCTION)
self._file_bundle_hash = h[obj]
return self._file_bundle_hash
def copy_bundle(
self, target_path: str, bundle_name: Union[str, None] = None
) -> "FileBundle":
if target_path == self.path:
raise Exception(f"Target path and current path are the same: {target_path}")
result = {}
for rel_path, item in self.included_files.items():
_target_path = os.path.join(target_path, rel_path)
new_fm = item.copy_file(_target_path)
result[rel_path] = new_fm
if bundle_name is None:
bundle_name = os.path.basename(target_path)
fb = FileBundle.create_from_file_models(
files=result,
bundle_name=bundle_name,
path=target_path,
sum_size=self.size,
# import_time=self.import_time,
)
if self._file_bundle_hash is not None:
fb._file_bundle_hash = self._file_bundle_hash
return fb
def create_renderable(self, **config: Any) -> RenderableType:
show_bundle_hash = config.get("show_bundle_hash", False)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key")
table.add_column("value", style="i")
table.add_row("bundle name", self.bundle_name)
# table.add_row("import_time", str(self.import_time))
table.add_row("number_of_files", str(self.number_of_files))
table.add_row("size", str(self.size))
if show_bundle_hash:
table.add_row("bundle_hash", str(self.file_bundle_hash))
content = self._create_content_table(**config)
table.add_row("included files", content)
return table
def _create_content_table(self, **render_config: Any) -> Table:
# show_content = render_config.get("show_content_preview", False)
max_no_included_files = render_config.get("max_no_files", 40)
table = Table(show_header=True, box=box.SIMPLE)
table.add_column("(relative) path")
table.add_column("size")
# if show_content:
# table.add_column("content preview")
if (
max_no_included_files < 0
or len(self.included_files) <= max_no_included_files
):
for f, model in self.included_files.items():
row = [f, str(model.size)]
table.add_row(*row)
else:
files = list(self.included_files.keys())
half = int((max_no_included_files - 1) / 2)
head = files[0:half]
tail = files[-1 * half :] # noqa
for rel_path in head:
model = self.included_files[rel_path]
row = [rel_path, str(model.size)]
table.add_row(*row)
table.add_row(" ... output skipped ...", "")
table.add_row(" ... output skipped ...", "")
for rel_path in tail:
model = self.included_files[rel_path]
row = [rel_path, str(model.size)]
table.add_row(*row)
return table
def __repr__(self):
return f"FileBundle(name={self.bundle_name})"
def __str__(self):
return self.__repr__()
bundle_name: str
pydantic-field
required
¶The name of this bundle.
file_bundle_hash: int
property
readonly
¶included_files: Dict[str, kiara.models.filesystem.FileModel]
pydantic-field
required
¶A map of all the included files, incl. their properties. Uses the relative path of each file as key.
number_of_files: int
pydantic-field
required
¶How many files are included in this bundle.
path: str
property
readonly
¶size: int
pydantic-field
required
¶The size of all files in this folder, combined.
copy_bundle(self, target_path, bundle_name=None)
¶Source code in kiara/models/filesystem.py
def copy_bundle(
self, target_path: str, bundle_name: Union[str, None] = None
) -> "FileBundle":
if target_path == self.path:
raise Exception(f"Target path and current path are the same: {target_path}")
result = {}
for rel_path, item in self.included_files.items():
_target_path = os.path.join(target_path, rel_path)
new_fm = item.copy_file(_target_path)
result[rel_path] = new_fm
if bundle_name is None:
bundle_name = os.path.basename(target_path)
fb = FileBundle.create_from_file_models(
files=result,
bundle_name=bundle_name,
path=target_path,
sum_size=self.size,
# import_time=self.import_time,
)
if self._file_bundle_hash is not None:
fb._file_bundle_hash = self._file_bundle_hash
return fb
create_from_file_models(files, bundle_name, path=None, sum_size=None)
classmethod
¶Source code in kiara/models/filesystem.py
@classmethod
def create_from_file_models(
cls,
files: Mapping[str, FileModel],
bundle_name: str,
path: Union[str, None] = None,
sum_size: Union[int, None] = None,
# import_time: Optional[datetime.datetime] = None,
) -> "FileBundle":
# if import_time:
# bundle_import_time = import_time
# else:
# bundle_import_time = datetime.datetime.now() # TODO: timezone
result: Dict[str, Any] = {}
result["included_files"] = files
# result["import_time"] = datetime.datetime.now().isoformat()
result["number_of_files"] = len(files)
result["bundle_name"] = bundle_name
# result["import_time"] = bundle_import_time
if sum_size is None:
sum_size = 0
for f in files.values():
sum_size = sum_size + f.size
result["size"] = sum_size
bundle = FileBundle(**result)
bundle._path = path
return bundle
create_renderable(self, **config)
¶Source code in kiara/models/filesystem.py
def create_renderable(self, **config: Any) -> RenderableType:
show_bundle_hash = config.get("show_bundle_hash", False)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key")
table.add_column("value", style="i")
table.add_row("bundle name", self.bundle_name)
# table.add_row("import_time", str(self.import_time))
table.add_row("number_of_files", str(self.number_of_files))
table.add_row("size", str(self.size))
if show_bundle_hash:
table.add_row("bundle_hash", str(self.file_bundle_hash))
content = self._create_content_table(**config)
table.add_row("included files", content)
return table
create_tmp_dir()
classmethod
¶Utility method to create a temp folder that gets deleted when kiara exits.
Source code in kiara/models/filesystem.py
@classmethod
def create_tmp_dir(self) -> Path:
"""Utility method to create a temp folder that gets deleted when kiara exits."""
temp_f = tempfile.mkdtemp()
def cleanup():
shutil.rmtree(temp_f, ignore_errors=True)
atexit.register(cleanup)
return Path(temp_f)
get_relative_path(self, file)
¶Source code in kiara/models/filesystem.py
def get_relative_path(self, file: FileModel):
return os.path.relpath(file.path, self.path)
import_folder(source, bundle_name=None, import_config=None)
classmethod
¶Source code in kiara/models/filesystem.py
@classmethod
def import_folder(
cls,
source: str,
bundle_name: Union[str, None] = None,
import_config: Union[None, Mapping[str, Any], FolderImportConfig] = None,
# import_time: Optional[datetime.datetime] = None,
) -> "FileBundle":
if not source:
raise ValueError("No source path provided.")
if not os.path.exists(os.path.realpath(source)):
raise ValueError(f"Path does not exist: {source}")
if not os.path.isdir(os.path.realpath(source)):
raise ValueError(f"Path is not a folder: {source}")
if source.endswith(os.path.sep):
source = source[0:-1]
abs_path = os.path.abspath(source)
if import_config is None:
_import_config = FolderImportConfig()
elif isinstance(import_config, Mapping):
_import_config = FolderImportConfig(**import_config)
elif isinstance(import_config, FolderImportConfig):
_import_config = import_config
else:
raise TypeError(
f"Invalid type for folder import config: {type(import_config)}."
)
included_files: Dict[str, FileModel] = {}
exclude_dirs = _import_config.exclude_dirs
invalid_extensions = _import_config.exclude_files
valid_extensions = _import_config.include_files
# if import_time:
# bundle_import_time = import_time
# else:
# bundle_import_time = datetime.datetime.now() # TODO: timezone
sum_size = 0
def include_file(filename: str) -> bool:
if invalid_extensions and any(
filename.endswith(ext) for ext in invalid_extensions
):
return False
if not valid_extensions:
return True
else:
return any(filename.endswith(ext) for ext in valid_extensions)
for root, dirnames, filenames in os.walk(abs_path, topdown=True):
if exclude_dirs:
dirnames[:] = [d for d in dirnames if d not in exclude_dirs]
for filename in [
f
for f in filenames
if os.path.isfile(os.path.join(root, f)) and include_file(f)
]:
full_path = os.path.join(root, filename)
rel_path = os.path.relpath(full_path, abs_path)
file_model = FileModel.load_file(full_path)
sum_size = sum_size + file_model.size
included_files[rel_path] = file_model
if bundle_name is None:
bundle_name = os.path.basename(source)
bundle = FileBundle.create_from_file_models(
files=included_files,
path=abs_path,
bundle_name=bundle_name,
sum_size=sum_size,
)
return bundle
read_text_file_contents(self, ignore_errors=False)
¶Source code in kiara/models/filesystem.py
def read_text_file_contents(self, ignore_errors: bool = False) -> Mapping[str, str]:
content_dict: Dict[str, str] = {}
def read_file(rel_path: str, full_path: str):
with open(full_path, encoding="utf-8") as f:
try:
content = f.read()
content_dict[rel_path] = content # type: ignore
except Exception as e:
if ignore_errors:
log_message(f"Can't read file: {e}")
logger.warning("ignore.file", path=full_path, reason=str(e))
else:
raise Exception(f"Can't read file (as text) '{full_path}: {e}")
# TODO: common ignore files and folders
for rel_path, f in self.included_files.items():
if f._path:
path = f._path
else:
path = self.get_relative_path(f)
read_file(rel_path=rel_path, full_path=path)
return content_dict
FileModel (KiaraModel)
pydantic-model
¶Describes properties for the 'file' value type.
Source code in kiara/models/filesystem.py
class FileModel(KiaraModel):
"""Describes properties for the 'file' value type."""
_kiara_model_id = "instance.data.file"
@classmethod
def load_file(
cls,
source: str,
file_name: Union[str, None] = None,
# import_time: Optional[datetime.datetime] = None,
):
"""Utility method to read metadata of a file from disk and optionally move it into a data archive location."""
import filetype
import mimetypes
if not source:
raise ValueError("No source path provided.")
if not os.path.exists(os.path.realpath(source)):
raise ValueError(f"Path does not exist: {source}")
if not os.path.isfile(os.path.realpath(source)):
raise ValueError(f"Path is not a file: {source}")
if file_name is None:
file_name = os.path.basename(source)
path: str = os.path.abspath(source)
# if import_time:
# file_import_time = import_time
# else:
# file_import_time = datetime.datetime.now() # TODO: timezone
file_stats = os.stat(path)
size = file_stats.st_size
r = mimetypes.guess_type(path)
if r[0] is not None:
mime_type = r[0]
else:
_mime_type = filetype.guess(path)
if not _mime_type:
mime_type = "application/octet-stream"
else:
mime_type = _mime_type.MIME
m = FileModel(
# import_time=file_import_time,
mime_type=mime_type,
size=size,
file_name=file_name,
)
m._path = path
return m
# import_time: datetime.datetime = Field(
# description="The time when the file was imported."
# )
mime_type: str = Field(description="The mime type of the file.")
file_name: str = Field("The name of the file.")
size: int = Field(description="The size of the file.")
_path: Union[str, None] = PrivateAttr(default=None)
_file_hash: Union[str, None] = PrivateAttr(default=None)
_file_cid: Union[CID, None] = PrivateAttr(default=None)
# @validator("path")
# def ensure_abs_path(cls, value):
# return os.path.abspath(value)
@property
def path(self) -> str:
if self._path is None:
raise Exception("File path not set for file model.")
return self._path
def _retrieve_data_to_hash(self) -> Any:
data = {
"file_name": self.file_name,
"file_cid": self.file_cid,
}
return data
# def get_id(self) -> str:
# return self.path
def get_category_alias(self) -> str:
return "instance.file_model"
def copy_file(self, target: str, new_name: Union[str, None] = None) -> "FileModel":
target_path: str = os.path.abspath(target)
os.makedirs(os.path.dirname(target_path), exist_ok=True)
shutil.copy2(self.path, target_path)
fm = FileModel.load_file(target, file_name=new_name)
if self._file_hash is not None:
fm._file_hash = self._file_hash
return fm
@property
def file_hash(self) -> str:
if self._file_hash is not None:
return self._file_hash
self._file_hash = str(self.file_cid)
return self._file_hash
@property
def file_cid(self) -> CID:
if self._file_cid is not None:
return self._file_cid
# TODO: auto-set codec?
self._file_cid = compute_cid_from_file(file=self.path, codec="raw")
return self._file_cid
@property
def file_name_without_extension(self) -> str:
return self.file_name.split(".")[0]
def read_text(self, max_lines: int = -1) -> str:
"""Read the content of a file."""
with open(self.path, "rt") as f:
if max_lines <= 0:
content = f.read()
else:
content = "".join((next(f) for x in range(max_lines)))
return content
def read_bytes(self, length: int = -1) -> bytes:
"""Read the content of a file."""
with open(self.path, "rb") as f:
if length <= 0:
content = f.read()
else:
content = f.read(length)
return content
def __repr__(self):
return f"FileModel(name={self.file_name})"
def __str__(self):
return self.__repr__()
file_cid: CID
property
readonly
¶file_hash: str
property
readonly
¶file_name: str
pydantic-field
¶file_name_without_extension: str
property
readonly
¶mime_type: str
pydantic-field
required
¶The mime type of the file.
path: str
property
readonly
¶size: int
pydantic-field
required
¶The size of the file.
copy_file(self, target, new_name=None)
¶Source code in kiara/models/filesystem.py
def copy_file(self, target: str, new_name: Union[str, None] = None) -> "FileModel":
target_path: str = os.path.abspath(target)
os.makedirs(os.path.dirname(target_path), exist_ok=True)
shutil.copy2(self.path, target_path)
fm = FileModel.load_file(target, file_name=new_name)
if self._file_hash is not None:
fm._file_hash = self._file_hash
return fm
get_category_alias(self)
¶Source code in kiara/models/filesystem.py
def get_category_alias(self) -> str:
return "instance.file_model"
load_file(source, file_name=None)
classmethod
¶Utility method to read metadata of a file from disk and optionally move it into a data archive location.
Source code in kiara/models/filesystem.py
@classmethod
def load_file(
cls,
source: str,
file_name: Union[str, None] = None,
# import_time: Optional[datetime.datetime] = None,
):
"""Utility method to read metadata of a file from disk and optionally move it into a data archive location."""
import filetype
import mimetypes
if not source:
raise ValueError("No source path provided.")
if not os.path.exists(os.path.realpath(source)):
raise ValueError(f"Path does not exist: {source}")
if not os.path.isfile(os.path.realpath(source)):
raise ValueError(f"Path is not a file: {source}")
if file_name is None:
file_name = os.path.basename(source)
path: str = os.path.abspath(source)
# if import_time:
# file_import_time = import_time
# else:
# file_import_time = datetime.datetime.now() # TODO: timezone
file_stats = os.stat(path)
size = file_stats.st_size
r = mimetypes.guess_type(path)
if r[0] is not None:
mime_type = r[0]
else:
_mime_type = filetype.guess(path)
if not _mime_type:
mime_type = "application/octet-stream"
else:
mime_type = _mime_type.MIME
m = FileModel(
# import_time=file_import_time,
mime_type=mime_type,
size=size,
file_name=file_name,
)
m._path = path
return m
read_bytes(self, length=-1)
¶Read the content of a file.
Source code in kiara/models/filesystem.py
def read_bytes(self, length: int = -1) -> bytes:
"""Read the content of a file."""
with open(self.path, "rb") as f:
if length <= 0:
content = f.read()
else:
content = f.read(length)
return content
read_text(self, max_lines=-1)
¶Read the content of a file.
Source code in kiara/models/filesystem.py
def read_text(self, max_lines: int = -1) -> str:
"""Read the content of a file."""
with open(self.path, "rt") as f:
if max_lines <= 0:
content = f.read()
else:
content = "".join((next(f) for x in range(max_lines)))
return content
FolderImportConfig (BaseModel)
pydantic-model
¶Source code in kiara/models/filesystem.py
class FolderImportConfig(BaseModel):
include_files: Union[List[str], None] = Field(
description="A list of strings, include all files where the filename ends with that string.",
default=None,
)
exclude_dirs: Union[List[str], None] = Field(
description="A list of strings, exclude all folders whose name ends with that string.",
default=None,
)
exclude_files: Union[List[str], None] = Field(
description=f"A list of strings, exclude all files that match those (takes precedence over 'include_files'). Defaults to: {DEFAULT_EXCLUDE_FILES}.",
default=DEFAULT_EXCLUDE_FILES,
)
exclude_dirs: List[str]
pydantic-field
¶A list of strings, exclude all folders whose name ends with that string.
exclude_files: List[str]
pydantic-field
¶A list of strings, exclude all files that match those (takes precedence over 'include_files'). Defaults to: ['.DS_Store'].
include_files: List[str]
pydantic-field
¶A list of strings, include all files where the filename ends with that string.
module
special
¶
Classes¶
KiaraModuleConfig (KiaraModel)
pydantic-model
¶Base class that describes the configuration a [KiaraModule][kiara.module.KiaraModule] class accepts.
This is stored in the _config_cls class attribute in each KiaraModule class.
There are two config options every KiaraModule supports:
constants, anddefaults
Constants are pre-set inputs, and users can't change them and an error is thrown if they try. Defaults are default values that override the schema defaults, and those can be overwritten by users. If both a constant and a default value is set for an input field, an error is thrown.
Source code in kiara/models/module/__init__.py
class KiaraModuleConfig(KiaraModel):
"""Base class that describes the configuration a [``KiaraModule``][kiara.module.KiaraModule] class accepts.
This is stored in the ``_config_cls`` class attribute in each ``KiaraModule`` class.
There are two config options every ``KiaraModule`` supports:
- ``constants``, and
- ``defaults``
Constants are pre-set inputs, and users can't change them and an error is thrown if they try. Defaults are default
values that override the schema defaults, and those can be overwritten by users. If both a constant and a default
value is set for an input field, an error is thrown.
"""
_kiara_model_id = "instance.module_config"
@classmethod
def requires_config(cls, config: Union[Mapping[str, Any], None] = None) -> bool:
"""Return whether this class can be used as-is, or requires configuration before an instance can be created."""
for field_name, field in cls.__fields__.items():
if field.required and field.default is None:
if config:
if config.get(field_name, None) is None:
return True
else:
return True
return False
_config_hash: str = PrivateAttr(default=None)
constants: Dict[str, Any] = Field(
default_factory=dict, description="Value constants for this module."
)
defaults: Dict[str, Any] = Field(
default_factory=dict, description="Value defaults for this module."
)
class Config:
extra = Extra.forbid
validate_assignment = True
def get(self, key: str) -> Any:
"""Get the value for the specified configuation key."""
if key not in self.__fields__:
raise Exception(
f"No config value '{key}' in module config class '{self.__class__.__name__}'."
)
return getattr(self, key)
def create_renderable(self, **config: Any) -> RenderableType:
my_table = Table(box=box.MINIMAL, show_header=False)
my_table.add_column("Field name", style="i")
my_table.add_column("Value")
for field in self.__fields__:
attr = getattr(self, field)
if isinstance(attr, str):
attr_str = attr
elif hasattr(attr, "create_renderable"):
attr_str = attr.create_renderable()
elif isinstance(attr, BaseModel):
attr_str = attr.json(option=orjson.orjson.OPT_INDENT_2)
else:
attr_str = str(attr)
my_table.add_row(field, attr_str)
return my_table
constants: Dict[str, Any]
pydantic-field
¶Value constants for this module.
defaults: Dict[str, Any]
pydantic-field
¶Value defaults for this module.
Config
¶create_renderable(self, **config)
¶Source code in kiara/models/module/__init__.py
def create_renderable(self, **config: Any) -> RenderableType:
my_table = Table(box=box.MINIMAL, show_header=False)
my_table.add_column("Field name", style="i")
my_table.add_column("Value")
for field in self.__fields__:
attr = getattr(self, field)
if isinstance(attr, str):
attr_str = attr
elif hasattr(attr, "create_renderable"):
attr_str = attr.create_renderable()
elif isinstance(attr, BaseModel):
attr_str = attr.json(option=orjson.orjson.OPT_INDENT_2)
else:
attr_str = str(attr)
my_table.add_row(field, attr_str)
return my_table
get(self, key)
¶Get the value for the specified configuation key.
Source code in kiara/models/module/__init__.py
def get(self, key: str) -> Any:
"""Get the value for the specified configuation key."""
if key not in self.__fields__:
raise Exception(
f"No config value '{key}' in module config class '{self.__class__.__name__}'."
)
return getattr(self, key)
requires_config(config=None)
classmethod
¶Return whether this class can be used as-is, or requires configuration before an instance can be created.
Source code in kiara/models/module/__init__.py
@classmethod
def requires_config(cls, config: Union[Mapping[str, Any], None] = None) -> bool:
"""Return whether this class can be used as-is, or requires configuration before an instance can be created."""
for field_name, field in cls.__fields__.items():
if field.required and field.default is None:
if config:
if config.get(field_name, None) is None:
return True
else:
return True
return False
Modules¶
destiniy
¶
Destiny (Manifest)
pydantic-model
¶A destiny is basically a link to a potential future transformation result involving one or several values as input.
It is immutable, once executed, each of the input values can only have one destiny with a specific alias. This is similar to what is usually called a 'future' in programming languages, but more deterministic, sorta.
Source code in kiara/models/module/destiniy.py
class Destiny(Manifest):
"""A destiny is basically a link to a potential future transformation result involving one or several values as input.
It is immutable, once executed, each of the input values can only have one destiny with a specific alias.
This is similar to what is usually called a 'future' in programming languages, but more deterministic, sorta.
"""
_kiara_model_id = "instance.destiny"
@classmethod
def create_from_values(
cls,
kiara: "Kiara",
destiny_alias: str,
values: Mapping[str, uuid.UUID],
manifest: Manifest,
result_field_name: Union[str, None] = None,
):
module = kiara.create_module(manifest=manifest)
if result_field_name is None:
if len(module.outputs_schema) != 1:
raise Exception(
f"Can't determine result field name for module, not provided, and multiple outputs available for module '{module.module_type_name}': {', '.join(module.outputs_schema.keys())}."
)
result_field_name = next(iter(module.outputs_schema.keys()))
result_schema = module.outputs_schema.get(result_field_name, None)
if result_schema is None:
raise Exception(
f"Can't determine result schema for module '{module.module_type_name}', result field '{result_field_name}' not available. Available field: {', '.join(module.outputs_schema.keys())}"
)
fixed_inputs = {}
deferred_inputs: Dict[str, None] = {}
for field in module.inputs_schema.keys():
if field in values.keys():
fixed_inputs[field] = values[field]
else:
deferred_inputs[field] = None
module_details = KiaraModuleInstance.from_module(module=module)
# TODO: check whether it'd be better to 'resolve' the module config, as this might change the resulting hash
destiny_id: uuid.UUID = ID_REGISTRY.generate(obj_type=Destiny)
destiny = Destiny(
destiny_id=destiny_id,
destiny_alias=destiny_alias,
module_details=module_details,
module_type=manifest.module_type,
module_config=manifest.module_config,
result_field_name=result_field_name,
result_schema=result_schema,
fixed_inputs=fixed_inputs,
inputs_schema=dict(module.inputs_schema),
deferred_inputs=deferred_inputs,
)
destiny._module = module
ID_REGISTRY.update_metadata(destiny_id, obj=destiny)
return destiny
destiny_id: uuid.UUID = Field(description="The id of this destiny.")
destiny_alias: str = Field(description="The path to (the) destiny.")
module_details: KiaraModuleInstance = Field(
description="The class of the underlying module."
)
fixed_inputs: Dict[str, uuid.UUID] = Field(
description="Inputs that are known in advance."
)
inputs_schema: Dict[str, ValueSchema] = Field(
description="The schemas of all deferred input fields."
)
deferred_inputs: Dict[str, Union[uuid.UUID, None]] = Field(
description="Potentially required external inputs that are needed for this destiny to materialize."
)
result_field_name: str = Field(description="The name of the result field.")
result_schema: ValueSchema = Field(description="The value schema of the result.")
result_value_id: Union[uuid.UUID, None] = Field(
description="The value id of the result."
)
_is_stored: bool = PrivateAttr(default=False)
_job_id: Union[uuid.UUID, None] = PrivateAttr(default=None)
_merged_inputs: Union[Dict[str, uuid.UUID], None] = PrivateAttr(default=None)
# _job_config_hash: Optional[int] = PrivateAttr(default=None)
_module: Union["KiaraModule", None] = PrivateAttr(default=None)
def _retrieve_id(self) -> str:
return str(self.destiny_id)
def _retrieve_data_to_hash(self) -> Any:
return self.destiny_id.bytes
# @property
# def job_config_hash(self) -> int:
# if self._job_config_hash is None:
# self._job_config_hash = self._retrieve_job_config_hash()
# return self._job_config_hash
@property
def merged_inputs(self) -> Mapping[str, uuid.UUID]:
if self._merged_inputs is not None:
return self._merged_inputs
result = copy.copy(self.fixed_inputs)
missing = []
for k in self.inputs_schema.keys():
if k in self.fixed_inputs.keys():
if k in self.deferred_inputs.keys():
raise Exception(
f"Destiny input field '{k}' present in both fixed and deferred inputs, this is invalid."
)
else:
continue
v = self.deferred_inputs.get(k, None)
if v is None or isinstance(v, SpecialValue):
missing.append(k)
else:
result[k] = v
if missing:
raise Exception(
f"Destiny not valid (yet), missing inputs: {', '.join(missing)}"
)
self._merged_inputs = result
return self._merged_inputs
@property
def module(self) -> "KiaraModule":
if self._module is None:
m_cls = self.module_details.get_class()
self._module = m_cls(module_config=self.module_config)
return self._module
# def _retrieve_job_config_hash(self) -> int:
# obj = {"module_config": self.manifest_data, "inputs": self.merged_inputs}
# return compute_cid(obj)
deferred_inputs: Dict[str, Optional[uuid.UUID]]
pydantic-field
required
¶Potentially required external inputs that are needed for this destiny to materialize.
destiny_alias: str
pydantic-field
required
¶The path to (the) destiny.
destiny_id: UUID
pydantic-field
required
¶The id of this destiny.
fixed_inputs: Dict[str, uuid.UUID]
pydantic-field
required
¶Inputs that are known in advance.
inputs_schema: Dict[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The schemas of all deferred input fields.
merged_inputs: Mapping[str, uuid.UUID]
property
readonly
¶module: KiaraModule
property
readonly
¶module_details: KiaraModuleInstance
pydantic-field
required
¶The class of the underlying module.
result_field_name: str
pydantic-field
required
¶The name of the result field.
result_schema: ValueSchema
pydantic-field
required
¶The value schema of the result.
result_value_id: UUID
pydantic-field
¶The value id of the result.
create_from_values(kiara, destiny_alias, values, manifest, result_field_name=None)
classmethod
¶Source code in kiara/models/module/destiniy.py
@classmethod
def create_from_values(
cls,
kiara: "Kiara",
destiny_alias: str,
values: Mapping[str, uuid.UUID],
manifest: Manifest,
result_field_name: Union[str, None] = None,
):
module = kiara.create_module(manifest=manifest)
if result_field_name is None:
if len(module.outputs_schema) != 1:
raise Exception(
f"Can't determine result field name for module, not provided, and multiple outputs available for module '{module.module_type_name}': {', '.join(module.outputs_schema.keys())}."
)
result_field_name = next(iter(module.outputs_schema.keys()))
result_schema = module.outputs_schema.get(result_field_name, None)
if result_schema is None:
raise Exception(
f"Can't determine result schema for module '{module.module_type_name}', result field '{result_field_name}' not available. Available field: {', '.join(module.outputs_schema.keys())}"
)
fixed_inputs = {}
deferred_inputs: Dict[str, None] = {}
for field in module.inputs_schema.keys():
if field in values.keys():
fixed_inputs[field] = values[field]
else:
deferred_inputs[field] = None
module_details = KiaraModuleInstance.from_module(module=module)
# TODO: check whether it'd be better to 'resolve' the module config, as this might change the resulting hash
destiny_id: uuid.UUID = ID_REGISTRY.generate(obj_type=Destiny)
destiny = Destiny(
destiny_id=destiny_id,
destiny_alias=destiny_alias,
module_details=module_details,
module_type=manifest.module_type,
module_config=manifest.module_config,
result_field_name=result_field_name,
result_schema=result_schema,
fixed_inputs=fixed_inputs,
inputs_schema=dict(module.inputs_schema),
deferred_inputs=deferred_inputs,
)
destiny._module = module
ID_REGISTRY.update_metadata(destiny_id, obj=destiny)
return destiny
jobs
¶
ActiveJob (KiaraModel)
pydantic-model
¶Source code in kiara/models/module/jobs.py
class ActiveJob(KiaraModel):
_kiara_model_id = "instance.active_job"
job_id: uuid.UUID = Field(description="The job id.")
job_config: JobConfig = Field(description="The job details.")
status: JobStatus = Field(
description="The current status of the job.", default=JobStatus.CREATED
)
job_log: JobLog = Field(description="The lob jog.")
submitted: datetime = Field(
description="When the job was submitted.", default_factory=datetime.now
)
started: Union[datetime, None] = Field(
description="When the job was started.", default=None
)
finished: Union[datetime, None] = Field(
description="When the job was finished.", default=None
)
results: Union[Dict[str, uuid.UUID], None] = Field(description="The result(s).")
error: Union[str, None] = Field(description="Potential error message.")
_exception: Union[Exception, None] = PrivateAttr(default=None)
def _retrieve_id(self) -> str:
return str(self.job_id)
def _retrieve_data_to_hash(self) -> Any:
return self.job_id.bytes
@property
def exception(self) -> Union[Exception, None]:
return self._exception
@property
def runtime(self) -> Union[float, None]:
if self.started is None or self.finished is None:
return None
runtime = self.finished - self.started
return runtime.total_seconds()
error: str
pydantic-field
¶Potential error message.
exception: Optional[Exception]
property
readonly
¶finished: datetime
pydantic-field
¶When the job was finished.
job_config: JobConfig
pydantic-field
required
¶The job details.
job_id: UUID
pydantic-field
required
¶The job id.
job_log: JobLog
pydantic-field
required
¶The lob jog.
results: Dict[str, uuid.UUID]
pydantic-field
¶The result(s).
runtime: Optional[float]
property
readonly
¶started: datetime
pydantic-field
¶When the job was started.
status: JobStatus
pydantic-field
¶The current status of the job.
submitted: datetime
pydantic-field
¶When the job was submitted.
ExecutionContext (KiaraModel)
pydantic-model
¶Source code in kiara/models/module/jobs.py
class ExecutionContext(KiaraModel):
_kiara_model_id = "instance.execution_context"
working_dir: str = Field(
description="The path of the working directory.", default_factory=os.getcwd
)
pipeline_dir: Union[str, None] = Field(
description="The path of the pipeline file that is being executed (if applicable)."
)
JobConfig (InputsManifest)
pydantic-model
¶Source code in kiara/models/module/jobs.py
class JobConfig(InputsManifest):
_kiara_model_id = "instance.job_config"
@classmethod
def create_from_module(
cls,
data_registry: "DataRegistry",
module: "KiaraModule",
inputs: Mapping[str, Any],
):
augmented = module.augment_module_inputs(inputs=inputs)
values = data_registry.create_valuemap(
data=augmented, schema=module.full_inputs_schema
)
invalid = values.check_invalid()
if invalid:
raise InvalidValuesException(invalid_values=invalid)
value_ids = values.get_all_value_ids()
return JobConfig.construct(
module_type=module.module_type_name,
module_config=module.config.dict(),
inputs=value_ids,
)
def _retrieve_data_to_hash(self) -> Any:
return {"manifest": self.manifest_cid, "inputs": self.inputs_cid}
create_from_module(data_registry, module, inputs)
classmethod
¶Source code in kiara/models/module/jobs.py
@classmethod
def create_from_module(
cls,
data_registry: "DataRegistry",
module: "KiaraModule",
inputs: Mapping[str, Any],
):
augmented = module.augment_module_inputs(inputs=inputs)
values = data_registry.create_valuemap(
data=augmented, schema=module.full_inputs_schema
)
invalid = values.check_invalid()
if invalid:
raise InvalidValuesException(invalid_values=invalid)
value_ids = values.get_all_value_ids()
return JobConfig.construct(
module_type=module.module_type_name,
module_config=module.config.dict(),
inputs=value_ids,
)
JobLog (BaseModel)
pydantic-model
¶Source code in kiara/models/module/jobs.py
class JobLog(BaseModel):
log: List[LogMessage] = Field(
description="The logs for this job.", default_factory=list
)
percent_finished: int = Field(
description="Describes how much of the job is finished. A negative number means the module does not support progress tracking.",
default=-1,
)
def add_log(self, msg: str, log_level: int = logging.DEBUG):
_msg = LogMessage(msg=msg, log_level=log_level)
self.log.append(_msg)
log: List[kiara.models.module.jobs.LogMessage]
pydantic-field
¶The logs for this job.
percent_finished: int
pydantic-field
¶Describes how much of the job is finished. A negative number means the module does not support progress tracking.
add_log(self, msg, log_level=10)
¶Source code in kiara/models/module/jobs.py
def add_log(self, msg: str, log_level: int = logging.DEBUG):
_msg = LogMessage(msg=msg, log_level=log_level)
self.log.append(_msg)
JobRecord (JobConfig)
pydantic-model
¶Source code in kiara/models/module/jobs.py
class JobRecord(JobConfig):
_kiara_model_id = "instance.job_record"
@classmethod
def from_active_job(self, kiara: "Kiara", active_job: ActiveJob):
assert active_job.status == JobStatus.SUCCESS
assert active_job.results is not None
job_details = JobRuntimeDetails.construct(
job_log=active_job.job_log,
submitted=active_job.submitted,
started=active_job.started, # type: ignore
finished=active_job.finished, # type: ignore
runtime=active_job.runtime, # type: ignore
)
inputs_data_cid = active_job.job_config.calculate_inputs_data_cid(
data_registry=kiara.data_registry
)
job_record = JobRecord(
job_id=active_job.job_id,
module_type=active_job.job_config.module_type,
module_config=active_job.job_config.module_config,
inputs=active_job.job_config.inputs,
outputs=active_job.results,
runtime_details=job_details,
environment_hashes=kiara.environment_registry.environment_hashes,
inputs_data_hash=str(inputs_data_cid)
if inputs_data_cid is not None
else None,
)
return job_record
job_id: uuid.UUID = Field(description="The globally unique id for this job.")
environment_hashes: Mapping[str, Mapping[str, str]] = Field(
description="Hashes for the environments this value was created in."
)
enviroments: Union[Mapping[str, Mapping[str, Any]], None] = Field(
description="Information about the environments this value was created in.",
default=None,
)
inputs_data_hash: Union[str, None] = Field(
description="A map of the hashes of this jobs inputs."
)
outputs: Dict[str, uuid.UUID] = Field(description="References to the job outputs.")
runtime_details: Union[JobRuntimeDetails, None] = Field(
description="Runtime details for the job."
)
job_metadata: Mapping[str, Any] = Field(
description="Optional metadata for this job.", default_factory=dict
)
_is_stored: bool = PrivateAttr(default=None)
_outputs_hash: Union[int, None] = PrivateAttr(default=None)
@validator("job_metadata", pre=True)
def validate_metadata(cls, value):
if value is None:
value = {}
return value
def _retrieve_data_to_hash(self) -> Any:
return {
"manifest": self.manifest_cid,
"inputs": self.inputs_cid,
"outputs": {k: v.bytes for k, v in self.outputs.items()},
}
def create_renderable(self, **config: Any) -> RenderableType:
from kiara.utils.output import extract_renderable
include = config.get("include", None)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Key", style="i")
table.add_column("Value")
for k in self.__fields__.keys():
if include is not None and k not in include:
continue
attr = getattr(self, k)
v = extract_renderable(attr)
table.add_row(k, v)
table.add_row("job hash", self.job_hash)
table.add_row("inputs hash", self.inputs_hash)
return table
# @property
# def outputs_hash(self) -> int:
#
# if self._outputs_hash is not None:
# return self._outputs_hash
#
# obj = self.outputs
# h = DeepHash(obj, hasher=KIARA_HASH_FUNCTION)
# self._outputs_hash = h[obj]
# return self._outputs_hash
enviroments: Mapping[str, Mapping[str, Any]]
pydantic-field
¶Information about the environments this value was created in.
environment_hashes: Mapping[str, Mapping[str, str]]
pydantic-field
required
¶Hashes for the environments this value was created in.
inputs_data_hash: str
pydantic-field
¶A map of the hashes of this jobs inputs.
job_id: UUID
pydantic-field
required
¶The globally unique id for this job.
job_metadata: Mapping[str, Any]
pydantic-field
¶Optional metadata for this job.
outputs: Dict[str, uuid.UUID]
pydantic-field
required
¶References to the job outputs.
runtime_details: JobRuntimeDetails
pydantic-field
¶Runtime details for the job.
create_renderable(self, **config)
¶Create a renderable for this module configuration.
Source code in kiara/models/module/jobs.py
def create_renderable(self, **config: Any) -> RenderableType:
from kiara.utils.output import extract_renderable
include = config.get("include", None)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Key", style="i")
table.add_column("Value")
for k in self.__fields__.keys():
if include is not None and k not in include:
continue
attr = getattr(self, k)
v = extract_renderable(attr)
table.add_row(k, v)
table.add_row("job hash", self.job_hash)
table.add_row("inputs hash", self.inputs_hash)
return table
from_active_job(kiara, active_job)
classmethod
¶Source code in kiara/models/module/jobs.py
@classmethod
def from_active_job(self, kiara: "Kiara", active_job: ActiveJob):
assert active_job.status == JobStatus.SUCCESS
assert active_job.results is not None
job_details = JobRuntimeDetails.construct(
job_log=active_job.job_log,
submitted=active_job.submitted,
started=active_job.started, # type: ignore
finished=active_job.finished, # type: ignore
runtime=active_job.runtime, # type: ignore
)
inputs_data_cid = active_job.job_config.calculate_inputs_data_cid(
data_registry=kiara.data_registry
)
job_record = JobRecord(
job_id=active_job.job_id,
module_type=active_job.job_config.module_type,
module_config=active_job.job_config.module_config,
inputs=active_job.job_config.inputs,
outputs=active_job.results,
runtime_details=job_details,
environment_hashes=kiara.environment_registry.environment_hashes,
inputs_data_hash=str(inputs_data_cid)
if inputs_data_cid is not None
else None,
)
return job_record
validate_metadata(value)
classmethod
¶Source code in kiara/models/module/jobs.py
@validator("job_metadata", pre=True)
def validate_metadata(cls, value):
if value is None:
value = {}
return value
JobRuntimeDetails (BaseModel)
pydantic-model
¶Source code in kiara/models/module/jobs.py
class JobRuntimeDetails(BaseModel):
# @classmethod
# def from_manifest(
# cls,
# manifest: Manifest,
# inputs: Mapping[str, Value],
# outputs: Mapping[str, Value],
# ):
#
# return JobRecord(
# module_type=manifest.module_type,
# module_config=manifest.module_config,
# inputs={k: v.value_id for k, v in inputs.items()},
# outputs={k: v.value_id for k, v in outputs.items()},
# )
job_log: JobLog = Field(description="The lob jog.")
submitted: datetime = Field(description="When the job was submitted.")
started: datetime = Field(description="When the job was started.")
finished: datetime = Field(description="When the job was finished.")
runtime: float = Field(description="The duration of the job.")
finished: datetime
pydantic-field
required
¶When the job was finished.
job_log: JobLog
pydantic-field
required
¶The lob jog.
runtime: float
pydantic-field
required
¶The duration of the job.
started: datetime
pydantic-field
required
¶When the job was started.
submitted: datetime
pydantic-field
required
¶When the job was submitted.
JobStatus (Enum)
¶
LogMessage (BaseModel)
pydantic-model
¶Source code in kiara/models/module/jobs.py
class LogMessage(BaseModel):
timestamp: datetime = Field(
description="The time the message was logged.", default_factory=datetime.now
)
log_level: int = Field(description="The log level.")
msg: str = Field(description="The log message")
manifest
¶
InputsManifest (Manifest)
pydantic-model
¶Source code in kiara/models/module/manifest.py
class InputsManifest(Manifest):
_kiara_model_id = "instance.manifest_with_inputs"
inputs: Mapping[str, uuid.UUID] = Field(
description="A map of all the input fields and value references."
)
_inputs_cid: Union[CID, None] = PrivateAttr(default=None)
_jobs_cid: Union[CID, None] = PrivateAttr(default=None)
_inputs_data_cid: Union[bool, CID, None] = PrivateAttr(default=None)
@validator("inputs")
def replace_none_values(cls, value):
result = {}
for k, v in value.items():
if v is None:
v = NONE_VALUE_ID
result[k] = v
return result
@property
def job_hash(self) -> str:
return str(self.job_cid)
@property
def job_cid(self) -> CID:
if self._jobs_cid is not None:
return self._jobs_cid
obj = {"manifest": self.manifest_cid, "inputs": self.inputs_cid}
_, self._jobs_cid = compute_cid(data=obj)
return self._jobs_cid
@property
def inputs_cid(self) -> CID:
if self._inputs_cid is not None:
return self._inputs_cid
_, cid = compute_cid(data={k: v.bytes for k, v in self.inputs.items()})
self._inputs_cid = cid
return self._inputs_cid
@property
def inputs_hash(self) -> str:
return str(self.inputs_cid)
def calculate_inputs_data_cid(
self, data_registry: "DataRegistry"
) -> Union[CID, None]:
if self._inputs_data_cid is not None:
if self._inputs_data_cid is False:
return None
return self._inputs_data_cid # type: ignore
data_hashes = {}
invalid = False
for k, v in self.inputs.items():
value = data_registry.get_value(v)
if value.value_hash == INVALID_HASH_MARKER:
invalid = True
break
data_hashes[k] = CID.decode(value.value_hash)
if invalid:
self._inputs_data_cid = False
return None
_, cid = compute_cid(data=data_hashes)
self._inputs_data_cid = cid
return cid
inputs: Mapping[str, uuid.UUID]
pydantic-field
required
¶A map of all the input fields and value references.
inputs_cid: CID
property
readonly
¶inputs_hash: str
property
readonly
¶job_cid: CID
property
readonly
¶job_hash: str
property
readonly
¶calculate_inputs_data_cid(self, data_registry)
¶Source code in kiara/models/module/manifest.py
def calculate_inputs_data_cid(
self, data_registry: "DataRegistry"
) -> Union[CID, None]:
if self._inputs_data_cid is not None:
if self._inputs_data_cid is False:
return None
return self._inputs_data_cid # type: ignore
data_hashes = {}
invalid = False
for k, v in self.inputs.items():
value = data_registry.get_value(v)
if value.value_hash == INVALID_HASH_MARKER:
invalid = True
break
data_hashes[k] = CID.decode(value.value_hash)
if invalid:
self._inputs_data_cid = False
return None
_, cid = compute_cid(data=data_hashes)
self._inputs_data_cid = cid
return cid
replace_none_values(value)
classmethod
¶Source code in kiara/models/module/manifest.py
@validator("inputs")
def replace_none_values(cls, value):
result = {}
for k, v in value.items():
if v is None:
v = NONE_VALUE_ID
result[k] = v
return result
Manifest (KiaraModel)
pydantic-model
¶A class to hold the type and configuration for a module instance.
Source code in kiara/models/module/manifest.py
class Manifest(KiaraModel):
"""A class to hold the type and configuration for a module instance."""
_kiara_model_id = "instance.manifest"
class Config:
extra = Extra.forbid
validate_all = True
_manifest_data: Union[Mapping[str, Any], None] = PrivateAttr(default=None)
_manifest_cid: Union[CID, None] = PrivateAttr(default=None)
module_type: str = Field(description="The module type.")
module_config: Mapping[str, Any] = Field(
default_factory=dict, description="The configuration for the module."
)
is_resolved: bool = Field(
description="Whether the configuration of this module was augmented with the module type defaults etc.",
default=False,
)
# python_class: PythonClass = Field(description="The python class that implements this module.")
# doc: DocumentationMetadataModel = Field(
# description="Documentation for this module instance.", default=None
# )
# @validator("module_config")
# def _validate_module_config(cls, value):
#
# return value
@property
def manifest_data(self):
"""The configuration data for this module instance."""
if self._manifest_data is not None:
return self._manifest_data
self._manifest_data = {
"module_type": self.module_type,
"module_config": self.module_config,
}
return self._manifest_data
@property
def manifest_cid(self) -> CID:
if self._manifest_cid is not None:
return self._manifest_cid
_, self._manifest_cid = compute_cid(self.manifest_data)
return self._manifest_cid
@property
def manifest_hash(self) -> str:
return str(self.manifest_cid)
def manifest_data_as_json(self):
return self.json(include={"module_type", "module_config"})
def _retrieve_data_to_hash(self) -> Any:
return self.manifest_data
def create_renderable(self, **config: Any) -> RenderableType:
"""Create a renderable for this module configuration."""
data = self.dict(exclude_none=True)
conf = Syntax(
orjson_dumps(data, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
return conf
def __repr__(self):
return f"{self.__class__.__name__}(module_type={self.module_type}, module_config={self.module_config})"
def __str__(self):
return self.__repr__()
is_resolved: bool
pydantic-field
¶Whether the configuration of this module was augmented with the module type defaults etc.
manifest_cid: CID
property
readonly
¶manifest_data
property
readonly
¶The configuration data for this module instance.
manifest_hash: str
property
readonly
¶module_config: Mapping[str, Any]
pydantic-field
¶The configuration for the module.
module_type: str
pydantic-field
required
¶The module type.
Config
¶create_renderable(self, **config)
¶Create a renderable for this module configuration.
Source code in kiara/models/module/manifest.py
def create_renderable(self, **config: Any) -> RenderableType:
"""Create a renderable for this module configuration."""
data = self.dict(exclude_none=True)
conf = Syntax(
orjson_dumps(data, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
return conf
manifest_data_as_json(self)
¶Source code in kiara/models/module/manifest.py
def manifest_data_as_json(self):
return self.json(include={"module_type", "module_config"})
operation
¶logger
¶
BaseOperationDetails (OperationDetails)
pydantic-model
¶Source code in kiara/models/module/operation.py
class BaseOperationDetails(OperationDetails):
_kiara_model_id = "instance.operation_details.base"
module_inputs_schema: Mapping[str, ValueSchema] = Field(
description="The input schemas of the module."
)
module_outputs_schema: Mapping[str, ValueSchema] = Field(
description="The output schemas of the module."
)
_op_schema: OperationSchema = PrivateAttr(default=None)
def get_operation_schema(self) -> OperationSchema:
if self._op_schema is not None:
return self._op_schema
self._op_schema = OperationSchema(
alias=self.operation_id,
inputs_schema=self.module_inputs_schema,
outputs_schema=self.module_outputs_schema,
)
return self._op_schema
module_inputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The input schemas of the module.
module_outputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The output schemas of the module.
get_operation_schema(self)
¶Source code in kiara/models/module/operation.py
def get_operation_schema(self) -> OperationSchema:
if self._op_schema is not None:
return self._op_schema
self._op_schema = OperationSchema(
alias=self.operation_id,
inputs_schema=self.module_inputs_schema,
outputs_schema=self.module_outputs_schema,
)
return self._op_schema
Filter (KiaraModel)
pydantic-model
¶Source code in kiara/models/module/operation.py
class Filter(KiaraModel):
operation: Operation = Field(
description="The underlying operation providing which does the filtering."
)
input_name: str = Field(
description="The input name to use for the dataset to filter."
)
output_name: str = Field(
description="The output name to use for the dataset to filter."
)
data_type: str = Field(description="The type of the dataset that gets filtered.")
data_type: str
pydantic-field
required
¶The type of the dataset that gets filtered.
input_name: str
pydantic-field
required
¶The input name to use for the dataset to filter.
operation: Operation
pydantic-field
required
¶The underlying operation providing which does the filtering.
output_name: str
pydantic-field
required
¶The output name to use for the dataset to filter.
ManifestOperationConfig (OperationConfig)
pydantic-model
¶Source code in kiara/models/module/operation.py
class ManifestOperationConfig(OperationConfig):
_kiara_model_id = "instance.operation_config.manifest"
module_type: str = Field(description="The module type.")
module_config: Dict[str, Any] = Field(
default_factory=dict, description="The configuration for the module."
)
def retrieve_module_type(self, kiara: "Kiara") -> str:
return self.module_type
def retrieve_module_config(self, kiara: "Kiara") -> Mapping[str, Any]:
return self.module_config
module_config: Dict[str, Any]
pydantic-field
¶The configuration for the module.
module_type: str
pydantic-field
required
¶The module type.
retrieve_module_config(self, kiara)
¶Source code in kiara/models/module/operation.py
def retrieve_module_config(self, kiara: "Kiara") -> Mapping[str, Any]:
return self.module_config
retrieve_module_type(self, kiara)
¶Source code in kiara/models/module/operation.py
def retrieve_module_type(self, kiara: "Kiara") -> str:
return self.module_type
Operation (Manifest)
pydantic-model
¶Source code in kiara/models/module/operation.py
class Operation(Manifest):
_kiara_model_id = "instance.operation"
@classmethod
def create_from_module(cls, module: KiaraModule, doc: Any = None) -> "Operation":
from kiara.operations.included_core_operations import (
CustomModuleOperationDetails,
)
op_id = f"{module.module_type_name}._{module.module_instance_cid}"
details = CustomModuleOperationDetails.create_from_module(module=module)
if doc is not None:
doc = DocumentationMetadataModel.create(doc)
else:
doc = DocumentationMetadataModel.from_class_doc(module.__class__)
operation = Operation(
module_type=module.module_type_name,
module_config=module.config.dict(),
operation_id=op_id,
operation_details=details,
module_details=KiaraModuleInstance.from_module(module),
doc=doc,
)
operation._module = module
return operation
operation_id: str = Field(description="The (unique) id of this operation.")
operation_details: OperationDetails = Field(
description="The operation specific details of this operation."
)
doc: DocumentationMetadataModel = Field(
description="Documentation for this operation."
)
module_details: KiaraModuleInstance = Field(
description="The class of the underlying module."
)
metadata: Mapping[str, Any] = Field(
description="Additional metadata for this operation.", default_factory=dict
)
_module: Union["KiaraModule", None] = PrivateAttr(default=None)
def _retrieve_data_to_hash(self) -> Any:
return {"operation_id": self.operation_id, "manifest": self.manifest_cid}
def _retrieve_id(self) -> str:
return self.operation_id
@property
def module(self) -> "KiaraModule":
if self._module is None:
m_cls = self.module_details.get_class()
self._module = m_cls(module_config=self.module_config)
return self._module
@property
def inputs_schema(self) -> Mapping[str, ValueSchema]:
return self.operation_details.inputs_schema
@property
def outputs_schema(self) -> Mapping[str, ValueSchema]:
return self.operation_details.outputs_schema
def prepare_job_config(
self, kiara: "Kiara", inputs: Mapping[str, Any]
) -> JobConfig:
augmented_inputs = (
self.operation_details.get_operation_schema().augment_module_inputs(
inputs=inputs
)
)
# module_inputs = self.operation_details.create_module_inputs(
# inputs=augmented_inputs
# )
job_config = kiara.job_registry.prepare_job_config(
manifest=self, inputs=augmented_inputs
)
return job_config
def run(self, kiara: "Kiara", inputs: Any) -> ValueMap:
logger.debug("run.operation", operation_id=self.operation_id)
job_config = self.prepare_job_config(kiara=kiara, inputs=inputs)
job_id = kiara.job_registry.execute_job(job_config=job_config)
outputs: ValueMap = kiara.job_registry.retrieve_result(job_id=job_id)
result = self.process_job_outputs(outputs=outputs)
return result
def process_job_outputs(self, outputs: ValueMap) -> ValueMap:
# op_outputs = self.operation_details.create_operation_outputs(outputs=outputs)
value_set = ValueMapReadOnly(value_items=outputs, values_schema=self.outputs_schema) # type: ignore
return value_set
# def run(self, _attach_lineage: bool = True, **inputs: Any) -> ValueMap:
#
# return self.module.run(_attach_lineage=_attach_lineage, **inputs)
# def create_html(self, **config) -> str:
#
# r = self.create_renderable(**config)
# p = Panel(r, title=f"Operation: {self.operation_id}", title_align="left")
# mime_bundle = p._repr_mimebundle_(include=[], exclude=[]) # type: ignore
# return mime_bundle["text/html"]
def create_renderable(self, **config: Any) -> RenderableType:
"""Create a printable overview of this operations details.
Available render_config options:
- 'include_full_doc' (default: True): whether to include the full documentation, or just a description
- 'include_src' (default: False): whether to include the module source code
"""
include_full_doc = config.get("include_full_doc", True)
include_src = config.get("include_src", False)
include_inputs = config.get("include_inputs", True)
include_outputs = config.get("include_outputs", True)
include_module_details = config.get("include_module_details", False)
table = Table(box=box.SIMPLE, show_header=False, show_lines=True)
table.add_column("Property", style="i")
table.add_column("Value")
if self.doc:
if include_full_doc:
table.add_row("Documentation", self.doc.full_doc)
else:
table.add_row("Description", self.doc.description)
# module_type_md = self.module.get_type_metadata()
if include_inputs:
inputs_table = create_table_from_field_schemas(
_add_required=True,
_add_default=True,
_show_header=True,
_constants=None,
fields=self.operation_details.inputs_schema,
)
table.add_row("Inputs", inputs_table)
if include_outputs:
outputs_table = create_table_from_field_schemas(
_add_required=False,
_add_default=False,
_show_header=True,
_constants=None,
fields=self.operation_details.outputs_schema,
)
table.add_row("Outputs", outputs_table)
if include_module_details:
table.add_row("Module type", self.module_type)
module_config = self.module.config.json(option=orjson.OPT_INDENT_2)
conf = Syntax(
module_config,
"json",
background_color="default",
)
table.add_row("Module config", conf)
from kiara.interfaces.python_api import ModuleTypeInfo
module_type_md = ModuleTypeInfo.create_from_type_class(
type_cls=self.module_details.get_class(), # type: ignore
kiara=None, # type: ignore
)
desc = module_type_md.documentation.description
module_md = module_type_md.create_renderable(
include_doc=False, include_src=False, include_config_schema=False
)
m_md = Group(desc, module_md)
table.add_row("Module metadata", m_md)
if include_src:
table.add_row("Source code", module_type_md.process_src)
return table
doc: DocumentationMetadataModel
pydantic-field
required
¶Documentation for this operation.
inputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶metadata: Mapping[str, Any]
pydantic-field
¶Additional metadata for this operation.
module: KiaraModule
property
readonly
¶module_details: KiaraModuleInstance
pydantic-field
required
¶The class of the underlying module.
operation_details: OperationDetails
pydantic-field
required
¶The operation specific details of this operation.
operation_id: str
pydantic-field
required
¶The (unique) id of this operation.
outputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶create_from_module(module, doc=None)
classmethod
¶Source code in kiara/models/module/operation.py
@classmethod
def create_from_module(cls, module: KiaraModule, doc: Any = None) -> "Operation":
from kiara.operations.included_core_operations import (
CustomModuleOperationDetails,
)
op_id = f"{module.module_type_name}._{module.module_instance_cid}"
details = CustomModuleOperationDetails.create_from_module(module=module)
if doc is not None:
doc = DocumentationMetadataModel.create(doc)
else:
doc = DocumentationMetadataModel.from_class_doc(module.__class__)
operation = Operation(
module_type=module.module_type_name,
module_config=module.config.dict(),
operation_id=op_id,
operation_details=details,
module_details=KiaraModuleInstance.from_module(module),
doc=doc,
)
operation._module = module
return operation
create_renderable(self, **config)
¶Create a printable overview of this operations details.
Available render_config options: - 'include_full_doc' (default: True): whether to include the full documentation, or just a description - 'include_src' (default: False): whether to include the module source code
Source code in kiara/models/module/operation.py
def create_renderable(self, **config: Any) -> RenderableType:
"""Create a printable overview of this operations details.
Available render_config options:
- 'include_full_doc' (default: True): whether to include the full documentation, or just a description
- 'include_src' (default: False): whether to include the module source code
"""
include_full_doc = config.get("include_full_doc", True)
include_src = config.get("include_src", False)
include_inputs = config.get("include_inputs", True)
include_outputs = config.get("include_outputs", True)
include_module_details = config.get("include_module_details", False)
table = Table(box=box.SIMPLE, show_header=False, show_lines=True)
table.add_column("Property", style="i")
table.add_column("Value")
if self.doc:
if include_full_doc:
table.add_row("Documentation", self.doc.full_doc)
else:
table.add_row("Description", self.doc.description)
# module_type_md = self.module.get_type_metadata()
if include_inputs:
inputs_table = create_table_from_field_schemas(
_add_required=True,
_add_default=True,
_show_header=True,
_constants=None,
fields=self.operation_details.inputs_schema,
)
table.add_row("Inputs", inputs_table)
if include_outputs:
outputs_table = create_table_from_field_schemas(
_add_required=False,
_add_default=False,
_show_header=True,
_constants=None,
fields=self.operation_details.outputs_schema,
)
table.add_row("Outputs", outputs_table)
if include_module_details:
table.add_row("Module type", self.module_type)
module_config = self.module.config.json(option=orjson.OPT_INDENT_2)
conf = Syntax(
module_config,
"json",
background_color="default",
)
table.add_row("Module config", conf)
from kiara.interfaces.python_api import ModuleTypeInfo
module_type_md = ModuleTypeInfo.create_from_type_class(
type_cls=self.module_details.get_class(), # type: ignore
kiara=None, # type: ignore
)
desc = module_type_md.documentation.description
module_md = module_type_md.create_renderable(
include_doc=False, include_src=False, include_config_schema=False
)
m_md = Group(desc, module_md)
table.add_row("Module metadata", m_md)
if include_src:
table.add_row("Source code", module_type_md.process_src)
return table
prepare_job_config(self, kiara, inputs)
¶Source code in kiara/models/module/operation.py
def prepare_job_config(
self, kiara: "Kiara", inputs: Mapping[str, Any]
) -> JobConfig:
augmented_inputs = (
self.operation_details.get_operation_schema().augment_module_inputs(
inputs=inputs
)
)
# module_inputs = self.operation_details.create_module_inputs(
# inputs=augmented_inputs
# )
job_config = kiara.job_registry.prepare_job_config(
manifest=self, inputs=augmented_inputs
)
return job_config
process_job_outputs(self, outputs)
¶Source code in kiara/models/module/operation.py
def process_job_outputs(self, outputs: ValueMap) -> ValueMap:
# op_outputs = self.operation_details.create_operation_outputs(outputs=outputs)
value_set = ValueMapReadOnly(value_items=outputs, values_schema=self.outputs_schema) # type: ignore
return value_set
run(self, kiara, inputs)
¶Source code in kiara/models/module/operation.py
def run(self, kiara: "Kiara", inputs: Any) -> ValueMap:
logger.debug("run.operation", operation_id=self.operation_id)
job_config = self.prepare_job_config(kiara=kiara, inputs=inputs)
job_id = kiara.job_registry.execute_job(job_config=job_config)
outputs: ValueMap = kiara.job_registry.retrieve_result(job_id=job_id)
result = self.process_job_outputs(outputs=outputs)
return result
OperationConfig (KiaraModel)
pydantic-model
¶Source code in kiara/models/module/operation.py
class OperationConfig(KiaraModel):
doc: DocumentationMetadataModel = Field(
description="Documentation for this operation."
)
@validator("doc", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
@abc.abstractmethod
def retrieve_module_type(self, kiara: "Kiara") -> str:
pass
@abc.abstractmethod
def retrieve_module_config(self, kiara: "Kiara") -> Mapping[str, Any]:
pass
doc: DocumentationMetadataModel
pydantic-field
required
¶Documentation for this operation.
retrieve_module_config(self, kiara)
¶Source code in kiara/models/module/operation.py
@abc.abstractmethod
def retrieve_module_config(self, kiara: "Kiara") -> Mapping[str, Any]:
pass
retrieve_module_type(self, kiara)
¶Source code in kiara/models/module/operation.py
@abc.abstractmethod
def retrieve_module_type(self, kiara: "Kiara") -> str:
pass
validate_doc(value)
classmethod
¶Source code in kiara/models/module/operation.py
@validator("doc", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
OperationDetails (KiaraModel)
pydantic-model
¶Source code in kiara/models/module/operation.py
class OperationDetails(KiaraModel):
_kiara_model_id = "instance.operation_details"
# inputs_map: Dict[str, str] = Field(description="A map with the operations input fields as keys, and the underlying modules input fields as values, used to translate input value maps.")
# outputs_map: Dict[str, str] = Field(description="A map with the operations input fields as keys, and the underlying modules input fields as values, used to translate input value maps.")
@classmethod
def create_operation_details(cls, **details: Any):
if PYDANTIC_USE_CONSTRUCT:
result = cls.construct(**details)
else:
result = cls(**details)
return result
operation_id: str = Field(description="The id of the operation.")
is_internal_operation: bool = Field(
description="Whether this operation is mainly used kiara-internally. Helps to hide it in UIs (operation lists etc.).",
default=False,
)
def _retrieve_id(self) -> str:
return self.operation_id
@property
def inputs_schema(self) -> Mapping[str, ValueSchema]:
"""The input schema for this module."""
return self.get_operation_schema().inputs_schema
@property
def outputs_schema(self) -> Mapping[str, ValueSchema]:
"""The input schema for this module."""
return self.get_operation_schema().outputs_schema
def get_operation_schema(self) -> OperationSchema:
raise NotImplementedError()
inputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶The input schema for this module.
is_internal_operation: bool
pydantic-field
¶Whether this operation is mainly used kiara-internally. Helps to hide it in UIs (operation lists etc.).
operation_id: str
pydantic-field
required
¶The id of the operation.
outputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶The input schema for this module.
create_operation_details(**details)
classmethod
¶Source code in kiara/models/module/operation.py
@classmethod
def create_operation_details(cls, **details: Any):
if PYDANTIC_USE_CONSTRUCT:
result = cls.construct(**details)
else:
result = cls(**details)
return result
get_operation_schema(self)
¶Source code in kiara/models/module/operation.py
def get_operation_schema(self) -> OperationSchema:
raise NotImplementedError()
OperationSchema (InputOutputObject)
¶Source code in kiara/models/module/operation.py
class OperationSchema(InputOutputObject):
def __init__(
self, alias: str, inputs_schema: ValueMapSchema, outputs_schema: ValueMapSchema
):
allow_empty_inputs = True
allow_empty_outputs = True
self._inputs_schema_static: ValueMapSchema = inputs_schema
self._outputs_schema_static: ValueMapSchema = outputs_schema
super().__init__(
alias=alias,
allow_empty_inputs_schema=allow_empty_inputs,
allow_empty_outputs_schema=allow_empty_outputs,
)
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return self._inputs_schema_static
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return self._outputs_schema_static
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/models/module/operation.py
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return self._inputs_schema_static
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/models/module/operation.py
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return self._outputs_schema_static
PipelineOperationConfig (OperationConfig)
pydantic-model
¶Source code in kiara/models/module/operation.py
class PipelineOperationConfig(OperationConfig):
_kiara_model_id = "instance.operation_config.pipeline"
pipeline_name: str = Field(description="The pipeline id.")
pipeline_config: Mapping[str, Any] = Field(description="The pipeline config data.")
module_map: Dict[str, Any] = Field(
description="A lookup map to resolves operation ids to module names/configs.",
default_factory=dict,
)
metadata: Mapping[str, Any] = Field(
description="Additional metadata for the pipeline.", default_factory=dict
)
@validator("pipeline_config")
def validate_pipeline_config(cls, value):
# TODO
assert isinstance(value, Mapping)
assert "steps" in value.keys()
return value
def retrieve_module_type(self, kiara: "Kiara") -> str:
return "pipeline"
def retrieve_module_config(self, kiara: "Kiara") -> Mapping[str, Any]:
# using _from_config here because otherwise we'd enter an infinite loop
pipeline_config = PipelineConfig._from_config(
pipeline_name=self.pipeline_name,
data=self.pipeline_config,
kiara=kiara,
module_map=self.module_map,
)
return pipeline_config.dict()
@property
def required_module_types(self) -> Iterable[str]:
return [step["module_type"] for step in self.pipeline_config["steps"]]
def __repr__(self):
return f"{self.__class__.__name__}(pipeline_name={self.pipeline_name} required_modules={list(self.required_module_types)} instance_id={self.instance_id}, category={self.model_type_d}, fields=[{', '.join(self.__fields__.keys())}])"
metadata: Mapping[str, Any]
pydantic-field
¶Additional metadata for the pipeline.
module_map: Dict[str, Any]
pydantic-field
¶A lookup map to resolves operation ids to module names/configs.
pipeline_config: Mapping[str, Any]
pydantic-field
required
¶The pipeline config data.
pipeline_name: str
pydantic-field
required
¶The pipeline id.
required_module_types: Iterable[str]
property
readonly
¶retrieve_module_config(self, kiara)
¶Source code in kiara/models/module/operation.py
def retrieve_module_config(self, kiara: "Kiara") -> Mapping[str, Any]:
# using _from_config here because otherwise we'd enter an infinite loop
pipeline_config = PipelineConfig._from_config(
pipeline_name=self.pipeline_name,
data=self.pipeline_config,
kiara=kiara,
module_map=self.module_map,
)
return pipeline_config.dict()
retrieve_module_type(self, kiara)
¶Source code in kiara/models/module/operation.py
def retrieve_module_type(self, kiara: "Kiara") -> str:
return "pipeline"
validate_pipeline_config(value)
classmethod
¶Source code in kiara/models/module/operation.py
@validator("pipeline_config")
def validate_pipeline_config(cls, value):
# TODO
assert isinstance(value, Mapping)
assert "steps" in value.keys()
return value
persistence
¶
ByteProvisioningStrategy (Enum)
¶
BytesAliasStructure (BaseModel)
pydantic-model
¶Source code in kiara/models/module/persistence.py
class BytesAliasStructure(BaseModel):
data_type: str = Field(description="The data type.")
data_type_config: Mapping[str, Any] = Field(description="The data type config.")
chunk_id_map: Mapping[str, List[str]] = Field(
description="References to byte arrays, Keys are field names, values are a list of hash-ids that the data is composed of.",
default_factory=dict,
)
chunk_id_map: Mapping[str, List[str]]
pydantic-field
¶References to byte arrays, Keys are field names, values are a list of hash-ids that the data is composed of.
data_type: str
pydantic-field
required
¶The data type.
data_type_config: Mapping[str, Any]
pydantic-field
required
¶The data type config.
BytesStructure (BaseModel)
pydantic-model
¶A data structure that
Source code in kiara/models/module/persistence.py
class BytesStructure(BaseModel):
"""A data structure that"""
data_type: str = Field(description="The data type.")
data_type_config: Mapping[str, Any] = Field(description="The data type config.")
chunk_map: Mapping[str, List[Union[str, bytes]]] = Field(
description="References to byte arrays, Keys are field names, values are a list of hash-ids that the data is composed of.",
default_factory=dict,
)
def provision_as_folder(self, copy_files: bool = False) -> Path:
pass
chunk_map: Mapping[str, List[Union[str, bytes]]]
pydantic-field
¶References to byte arrays, Keys are field names, values are a list of hash-ids that the data is composed of.
data_type: str
pydantic-field
required
¶The data type.
data_type_config: Mapping[str, Any]
pydantic-field
required
¶The data type config.
provision_as_folder(self, copy_files=False)
¶Source code in kiara/models/module/persistence.py
def provision_as_folder(self, copy_files: bool = False) -> Path:
pass
pipeline
special
¶
PipelineConfig (KiaraModuleConfig)
pydantic-model
¶A class to hold the configuration for a [PipelineModule][kiara.pipeline.module.PipelineModule].
If you want to control the pipeline input and output names, you need to have to provide a map that uses the autogenerated field name ([step_id]__[alias] -- 2 underscores!!) as key, and the desired field name as value. The reason that schema for the autogenerated field names exist is that it's hard to ensure the uniqueness of each field; some steps can have the same input field names, but will need different input values. In some cases, some inputs of different steps need the same input. Those sorts of things. So, to make sure that we always use the right values, I chose to implement a conservative default approach, accepting that in some cases the user will be prompted for duplicate inputs for the same value.
To remedy that, the pipeline creator has the option to manually specify a mapping to rename some or all of the input/output fields.
Further, because in a lot of cases there won't be any overlapping fields, the creator can specify auto,
in which case Kiara will automatically create a mapping that tries to map autogenerated field names
to the shortest possible names for each case.
Examples:
Configuration for a pipeline module that functions as a nand logic gate (in Python):
and_step = PipelineStepConfig(module_type="and", step_id="and")
not_step = PipelineStepConfig(module_type="not", step_id="not", input_links={"a": ["and.y"]}
nand_p_conf = PipelineConfig(doc="Returns 'False' if both inputs are 'True'.",
steps=[and_step, not_step],
input_aliases={
"and__a": "a",
"and__b": "b"
},
output_aliases={
"not__y": "y"
}}
Or, the same thing in json:
{
"module_type_name": "nand",
"doc": "Returns 'False' if both inputs are 'True'.",
"steps": [
{
"module_type": "and",
"step_id": "and"
},
{
"module_type": "not",
"step_id": "not",
"input_links": {
"a": "and.y"
}
}
],
"input_aliases": {
"and__a": "a",
"and__b": "b"
},
"output_aliases": {
"not__y": "y"
}
}
Source code in kiara/models/module/pipeline/__init__.py
class PipelineConfig(KiaraModuleConfig):
"""A class to hold the configuration for a [PipelineModule][kiara.pipeline.module.PipelineModule].
If you want to control the pipeline input and output names, you need to have to provide a map that uses the
autogenerated field name ([step_id]__[alias] -- 2 underscores!!) as key, and the desired field name
as value. The reason that schema for the autogenerated field names exist is that it's hard to ensure
the uniqueness of each field; some steps can have the same input field names, but will need different input
values. In some cases, some inputs of different steps need the same input. Those sorts of things.
So, to make sure that we always use the right values, I chose to implement a conservative default approach,
accepting that in some cases the user will be prompted for duplicate inputs for the same value.
To remedy that, the pipeline creator has the option to manually specify a mapping to rename some or all of
the input/output fields.
Further, because in a lot of cases there won't be any overlapping fields, the creator can specify ``auto``,
in which case *Kiara* will automatically create a mapping that tries to map autogenerated field names
to the shortest possible names for each case.
Examples:
Configuration for a pipeline module that functions as a ``nand`` logic gate (in Python):
``` python
and_step = PipelineStepConfig(module_type="and", step_id="and")
not_step = PipelineStepConfig(module_type="not", step_id="not", input_links={"a": ["and.y"]}
nand_p_conf = PipelineConfig(doc="Returns 'False' if both inputs are 'True'.",
steps=[and_step, not_step],
input_aliases={
"and__a": "a",
"and__b": "b"
},
output_aliases={
"not__y": "y"
}}
```
Or, the same thing in json:
``` json
{
"module_type_name": "nand",
"doc": "Returns 'False' if both inputs are 'True'.",
"steps": [
{
"module_type": "and",
"step_id": "and"
},
{
"module_type": "not",
"step_id": "not",
"input_links": {
"a": "and.y"
}
}
],
"input_aliases": {
"and__a": "a",
"and__b": "b"
},
"output_aliases": {
"not__y": "y"
}
}
```
"""
_kiara_model_id = "instance.module_config.pipeline"
@classmethod
def from_file(
cls,
path: str,
kiara: Union["Kiara", None] = None,
# module_map: Optional[Mapping[str, Any]] = None,
):
data = get_data_from_file(path)
pipeline_name = data.pop("pipeline_name", None)
if pipeline_name is None:
pipeline_name = os.path.basename(path)
pipeline_dir = os.path.abspath(os.path.dirname(path))
execution_context = ExecutionContext(pipeline_dir=pipeline_dir)
return cls.from_config(
pipeline_name=pipeline_name,
data=data,
kiara=kiara,
execution_context=execution_context,
)
@classmethod
def from_config(
cls,
pipeline_name: str,
data: Mapping[str, Any],
kiara: Union["Kiara", None] = None,
module_map: Union[Mapping[str, Any], None] = None,
execution_context: Union[ExecutionContext, None] = None,
auto_step_ids: bool = False,
):
if kiara is None:
from kiara.context import Kiara
kiara = Kiara.instance()
if not kiara.operation_registry.is_initialized:
kiara.operation_registry.operations # noqa
if execution_context is None:
execution_context = ExecutionContext()
config = cls._from_config(
pipeline_name=pipeline_name,
data=data,
kiara=kiara,
module_map=module_map,
execution_context=execution_context,
auto_step_ids=auto_step_ids,
)
return config
@classmethod
def _from_config(
cls,
pipeline_name: str,
data: Mapping[str, Any],
kiara: "Kiara",
module_map: Union[Mapping[str, Any], None] = None,
execution_context: Union[ExecutionContext, None] = None,
auto_step_ids: bool = False,
):
if execution_context is None:
execution_context = ExecutionContext()
repl_dict = execution_context.dict()
data = dict(data)
steps = data.pop("steps")
steps = PipelineStep.create_steps(
*steps, kiara=kiara, module_map=module_map, auto_step_ids=auto_step_ids
)
data["steps"] = steps
if not data.get("input_aliases"):
data["input_aliases"] = create_input_alias_map(steps)
if not data.get("output_aliases"):
data["output_aliases"] = create_output_alias_map(steps)
if "defaults" in data.keys():
defaults = data.pop("defaults")
replaced = replace_var_names_in_obj(defaults, repl_dict=repl_dict)
data["defaults"] = replaced
if "constants" in data.keys():
constants = data.pop("constants")
replaced = replace_var_names_in_obj(constants, repl_dict=repl_dict)
data["constants"] = replaced
if "inputs" in data.keys():
inputs = data.pop("inputs")
replaced = replace_var_names_in_obj(inputs, repl_dict=repl_dict)
data["inputs"] = replaced
result = cls(pipeline_name=pipeline_name, **data)
return result
class Config:
extra = Extra.ignore
validate_assignment = True
pipeline_name: str = Field(description="The name of this pipeline.")
steps: List[PipelineStep] = Field(
description="A list of steps/modules of this pipeline, and their connections.",
)
input_aliases: Dict[str, str] = Field(
description="A map of input aliases, with the location of the input (in the format '[step_id].[input_field]') as key, and the pipeline input field name as value.",
)
output_aliases: Dict[str, str] = Field(
description="A map of output aliases, with the location of the output (in the format '[step_id].[output_field]') as key, and the pipeline output field name as value.",
)
doc: DocumentationMetadataModel = Field(
default="-- n/a --", description="Documentation about what the pipeline does."
)
context: Dict[str, Any] = Field(
default_factory=dict, description="Metadata for this workflow."
)
_structure: Union["PipelineStructure", None] = PrivateAttr(default=None)
@validator("doc", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
@validator("steps", pre=True)
def _validate_steps(cls, v):
# if not v:
# raise ValueError(f"Invalid type for 'steps' value: {type(v)}")
steps = []
for step in v:
if not step:
raise ValueError("No step data provided.")
if isinstance(step, PipelineStep):
steps.append(step)
elif isinstance(step, Mapping):
steps.append(PipelineStep(**step))
else:
raise TypeError(step)
return steps
@property
def structure(self) -> "PipelineStructure":
if self._structure is not None:
return self._structure
from kiara.models.module.pipeline.structure import PipelineStructure
self._structure = PipelineStructure(pipeline_config=self)
return self._structure
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
table.add_row("doc", self.doc.full_doc)
table.add_row("structure", self.structure)
return table
# return create_table_from_model_object(self, exclude_fields={"steps"})
return create_table_from_model_object(self)
# def create_input_alias_map(self) -> Dict[str, str]:
#
# aliases: Dict[str, List[str]] = {}
# for step in self.steps:
# field_names = step.module.input_names
# for field_name in field_names:
# aliases.setdefault(field_name, []).append(step.step_id)
#
# result: Dict[str, str] = {}
# for field_name, step_ids in aliases.items():
# for step_id in step_ids:
# generated = generate_pipeline_endpoint_name(step_id, field_name)
# result[generated] = generated
#
# return result
#
# def create_output_alias_map(self) -> Dict[str, str]:
#
# aliases: Dict[str, List[str]] = {}
# for step in self.steps:
# field_names = step.module.input_names
# for field_name in field_names:
# aliases.setdefault(field_name, []).append(step.step_id)
#
# result: Dict[str, str] = {}
# for field_name, step_ids in aliases.items():
# for step_id in step_ids:
# generated = generate_pipeline_endpoint_name(step_id, field_name)
# result[generated] = generated
#
# return result
context: Dict[str, Any]
pydantic-field
¶Metadata for this workflow.
doc: DocumentationMetadataModel
pydantic-field
¶Documentation about what the pipeline does.
input_aliases: Dict[str, str]
pydantic-field
required
¶A map of input aliases, with the location of the input (in the format '[step_id].[input_field]') as key, and the pipeline input field name as value.
output_aliases: Dict[str, str]
pydantic-field
required
¶A map of output aliases, with the location of the output (in the format '[step_id].[output_field]') as key, and the pipeline output field name as value.
pipeline_name: str
pydantic-field
required
¶The name of this pipeline.
steps: List[kiara.models.module.pipeline.PipelineStep]
pydantic-field
required
¶A list of steps/modules of this pipeline, and their connections.
structure: PipelineStructure
property
readonly
¶
Config
¶create_renderable(self, **config)
¶Source code in kiara/models/module/pipeline/__init__.py
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
table.add_row("doc", self.doc.full_doc)
table.add_row("structure", self.structure)
return table
# return create_table_from_model_object(self, exclude_fields={"steps"})
return create_table_from_model_object(self)
from_config(pipeline_name, data, kiara=None, module_map=None, execution_context=None, auto_step_ids=False)
classmethod
¶Source code in kiara/models/module/pipeline/__init__.py
@classmethod
def from_config(
cls,
pipeline_name: str,
data: Mapping[str, Any],
kiara: Union["Kiara", None] = None,
module_map: Union[Mapping[str, Any], None] = None,
execution_context: Union[ExecutionContext, None] = None,
auto_step_ids: bool = False,
):
if kiara is None:
from kiara.context import Kiara
kiara = Kiara.instance()
if not kiara.operation_registry.is_initialized:
kiara.operation_registry.operations # noqa
if execution_context is None:
execution_context = ExecutionContext()
config = cls._from_config(
pipeline_name=pipeline_name,
data=data,
kiara=kiara,
module_map=module_map,
execution_context=execution_context,
auto_step_ids=auto_step_ids,
)
return config
from_file(path, kiara=None)
classmethod
¶Source code in kiara/models/module/pipeline/__init__.py
@classmethod
def from_file(
cls,
path: str,
kiara: Union["Kiara", None] = None,
# module_map: Optional[Mapping[str, Any]] = None,
):
data = get_data_from_file(path)
pipeline_name = data.pop("pipeline_name", None)
if pipeline_name is None:
pipeline_name = os.path.basename(path)
pipeline_dir = os.path.abspath(os.path.dirname(path))
execution_context = ExecutionContext(pipeline_dir=pipeline_dir)
return cls.from_config(
pipeline_name=pipeline_name,
data=data,
kiara=kiara,
execution_context=execution_context,
)
validate_doc(value)
classmethod
¶Source code in kiara/models/module/pipeline/__init__.py
@validator("doc", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
PipelineStep (Manifest)
pydantic-model
¶A step within a pipeline-structure, includes information about it's connection(s) and other metadata.
Source code in kiara/models/module/pipeline/__init__.py
class PipelineStep(Manifest):
"""A step within a pipeline-structure, includes information about it's connection(s) and other metadata."""
_kiara_model_id = "instance.pipeline_step"
class Config:
validate_assignment = True
extra = Extra.forbid
@classmethod
def create_steps(
cls,
*steps: Union["PipelineStep", Mapping[str, Any]],
kiara: "Kiara",
module_map: Union[Mapping[str, Any], None] = None,
auto_step_ids: bool = False,
) -> List["PipelineStep"]:
if module_map is None:
module_map = {}
else:
module_map = dict(module_map)
# if kiara.operation_registry.is_initialized:
# module_map.update(kiara.operation_registry.get_module_map())
result: List[PipelineStep] = []
step_ids: List[str] = []
for step in steps:
if not isinstance(step, PipelineStep):
module_type = step.get("module_type", None)
if not module_type:
raise ValueError("Can't create step, no 'module_type' specified.")
module_config = step.get("module_config", {})
if module_type not in kiara.module_type_names:
if module_type in module_map.keys():
resolved_module_type = module_map[module_type]["module_type"]
resolved_module_config = module_map[module_type][
"module_config"
]
manifest = kiara.create_manifest(
module_or_operation=resolved_module_type,
config=resolved_module_config,
)
elif (
kiara.operation_registry.is_initialized
and module_type in kiara.operation_registry.operation_ids
):
op = kiara.operation_registry.operations[module_type]
resolved_module_type = op.module_type
resolved_module_config = op.module_config
manifest = kiara.create_manifest(
module_or_operation=resolved_module_type,
config=resolved_module_config,
)
else:
raise Exception(f"Can't resolve module type: {module_type}")
else:
manifest = kiara.create_manifest(
module_or_operation=module_type, config=module_config
)
resolved_module_type = module_type
resolved_module_config = module_config
module = kiara.create_module(manifest=manifest)
step_id = step.get("step_id", None)
if not step_id:
if not auto_step_ids:
raise ValueError("Can't create step, no 'step_id' specified.")
else:
step_id = find_free_id(
slugify(manifest.module_type, separator="_"),
current_ids=step_ids,
)
if step_id in step_ids:
raise ValueError(
f"Can't create step: duplicate step id '{step_id}'."
)
step_ids.append(step_id)
input_links = {}
for input_field, sources in step.get("input_links", {}).items():
if isinstance(sources, str):
sources = [sources]
input_links[input_field] = sources
doc = step.get("doc", None)
# TODO: do we really need the deepcopy here?
_s = PipelineStep(
step_id=step_id,
module_type=resolved_module_type,
module_config=dict(resolved_module_config),
input_links=input_links, # type: ignore
doc=doc,
module_details=KiaraModuleInstance.from_module(module=module),
)
_s._module = module
result.append(_s)
else:
result.append(step)
return result
@validator("step_id")
def _validate_step_id(cls, v):
assert isinstance(v, str)
if "." in v:
raise ValueError("Step ids can't contain '.' characters.")
return v
step_id: str = Field(
description="Locally unique id (within a pipeline) of this step."
)
module_type: str = Field(description="The module type.")
module_config: Dict[str, Any] = Field(
description="The module config.", default_factory=dict
)
# required: bool = Field(
# description="Whether this step is required within the workflow.\n\nIn some cases, when none of the pipeline outputs have a required input that connects to a step, then it is not necessary for this step to have been executed, even if it is placed before a step in the execution hierarchy. This also means that the pipeline inputs that are connected to this step might not be required.",
# default=True,
# )
# processing_stage: Optional[int] = Field(
# default=None,
# description="The stage number this step is executed within the pipeline.",
# )
input_links: Mapping[str, List[StepValueAddress]] = Field(
description="The links that connect to inputs of the module. Keys are field names, value(s) are connected outputs.",
default_factory=dict,
)
module_details: KiaraModuleInstance = Field(
description="The class of the underlying module."
)
doc: DocumentationMetadataModel = Field(
description="A description what this step does."
)
_module: Union["KiaraModule", None] = PrivateAttr(default=None)
@root_validator(pre=True)
def create_step_id(cls, values):
if "module_type" not in values:
raise ValueError("No 'module_type' specified.")
if "step_id" not in values or not values["step_id"]:
values["step_id"] = slugify(values["module_type"], separator="_")
return values
@validator("doc", pre=True)
def validate_doc(cls, value):
doc = DocumentationMetadataModel.create(value)
return doc
@validator("step_id")
def ensure_valid_id(cls, v):
# TODO: check with regex
if "." in v or " " in v:
raise ValueError(
f"Step id can't contain special characters or whitespaces: {v}"
)
return v
@validator("module_config", pre=True)
def ensure_dict(cls, v):
if v is None:
v = {}
return v
@validator("input_links", pre=True)
def ensure_input_links_valid(cls, v):
if v is None:
v = {}
result = {}
for input_name, output in v.items():
input_links = ensure_step_value_addresses(
default_field_name=input_name, link=output
)
result[input_name] = input_links
return result
@property
def module(self) -> "KiaraModule":
if self._module is None:
m_cls = self.module_details.get_class()
self._module = m_cls(module_config=self.module_config)
return self._module
def __repr__(self):
return f"{self.__class__.__name__}(step_id={self.step_id} module_type={self.module_type})"
def __str__(self):
return f"step: {self.step_id} (module: {self.module_type})"
def create_renderable(self, **config: Any) -> RenderableType:
in_panel = config.get("in_panel", None)
if in_panel is None:
if is_jupyter():
in_panel = True
else:
in_panel = False
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
if self.doc.is_set:
table.add_row("", Markdown(self.doc.full_doc))
table.add_row("step_id", self.step_id)
table.add_row("module type", self.module_type)
if not module_config_is_empty(self.module_config):
mc = dict(self.module_config)
if not mc.get("defaults", None):
mc.pop("defaults", None)
if not mc.get("constants", None):
mc.pop("constants", None)
config_str = orjson_dumps(mc, option=orjson.OPT_INDENT_2)
table.add_row(
"module_config",
Syntax(config_str, "json", background_color="default", theme="default"),
)
module_doc = DocumentationMetadataModel.from_class_doc(self.module.__class__)
table.add_row("module doc", Markdown(module_doc.full_doc))
inputs = create_table_from_field_schemas(
_add_default=True,
_add_required=True,
_show_header=True,
fields={
f"{self.step_id}.{k}": v for k, v in self.module.inputs_schema.items()
},
)
table.add_row("inputs", inputs)
outputs = create_table_from_field_schemas(
_add_default=False,
_add_required=False,
_show_header=True,
fields={
f"{self.step_id}.{k}": v for k, v in self.module.outputs_schema.items()
},
)
table.add_row("outputs", outputs)
if in_panel:
return Panel(table, title=f"Step: {self.step_id}", title_align="left")
else:
return table
doc: DocumentationMetadataModel
pydantic-field
required
¶A description what this step does.
input_links: Mapping[str, List[kiara.models.module.pipeline.value_refs.StepValueAddress]]
pydantic-field
¶The links that connect to inputs of the module. Keys are field names, value(s) are connected outputs.
module: KiaraModule
property
readonly
¶module_details: KiaraModuleInstance
pydantic-field
required
¶The class of the underlying module.
step_id: str
pydantic-field
required
¶Locally unique id (within a pipeline) of this step.
Config
¶create_renderable(self, **config)
¶Create a renderable for this module configuration.
Source code in kiara/models/module/pipeline/__init__.py
def create_renderable(self, **config: Any) -> RenderableType:
in_panel = config.get("in_panel", None)
if in_panel is None:
if is_jupyter():
in_panel = True
else:
in_panel = False
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
if self.doc.is_set:
table.add_row("", Markdown(self.doc.full_doc))
table.add_row("step_id", self.step_id)
table.add_row("module type", self.module_type)
if not module_config_is_empty(self.module_config):
mc = dict(self.module_config)
if not mc.get("defaults", None):
mc.pop("defaults", None)
if not mc.get("constants", None):
mc.pop("constants", None)
config_str = orjson_dumps(mc, option=orjson.OPT_INDENT_2)
table.add_row(
"module_config",
Syntax(config_str, "json", background_color="default", theme="default"),
)
module_doc = DocumentationMetadataModel.from_class_doc(self.module.__class__)
table.add_row("module doc", Markdown(module_doc.full_doc))
inputs = create_table_from_field_schemas(
_add_default=True,
_add_required=True,
_show_header=True,
fields={
f"{self.step_id}.{k}": v for k, v in self.module.inputs_schema.items()
},
)
table.add_row("inputs", inputs)
outputs = create_table_from_field_schemas(
_add_default=False,
_add_required=False,
_show_header=True,
fields={
f"{self.step_id}.{k}": v for k, v in self.module.outputs_schema.items()
},
)
table.add_row("outputs", outputs)
if in_panel:
return Panel(table, title=f"Step: {self.step_id}", title_align="left")
else:
return table
create_step_id(values)
classmethod
¶Source code in kiara/models/module/pipeline/__init__.py
@root_validator(pre=True)
def create_step_id(cls, values):
if "module_type" not in values:
raise ValueError("No 'module_type' specified.")
if "step_id" not in values or not values["step_id"]:
values["step_id"] = slugify(values["module_type"], separator="_")
return values
create_steps(*steps, *, kiara, module_map=None, auto_step_ids=False)
classmethod
¶Source code in kiara/models/module/pipeline/__init__.py
@classmethod
def create_steps(
cls,
*steps: Union["PipelineStep", Mapping[str, Any]],
kiara: "Kiara",
module_map: Union[Mapping[str, Any], None] = None,
auto_step_ids: bool = False,
) -> List["PipelineStep"]:
if module_map is None:
module_map = {}
else:
module_map = dict(module_map)
# if kiara.operation_registry.is_initialized:
# module_map.update(kiara.operation_registry.get_module_map())
result: List[PipelineStep] = []
step_ids: List[str] = []
for step in steps:
if not isinstance(step, PipelineStep):
module_type = step.get("module_type", None)
if not module_type:
raise ValueError("Can't create step, no 'module_type' specified.")
module_config = step.get("module_config", {})
if module_type not in kiara.module_type_names:
if module_type in module_map.keys():
resolved_module_type = module_map[module_type]["module_type"]
resolved_module_config = module_map[module_type][
"module_config"
]
manifest = kiara.create_manifest(
module_or_operation=resolved_module_type,
config=resolved_module_config,
)
elif (
kiara.operation_registry.is_initialized
and module_type in kiara.operation_registry.operation_ids
):
op = kiara.operation_registry.operations[module_type]
resolved_module_type = op.module_type
resolved_module_config = op.module_config
manifest = kiara.create_manifest(
module_or_operation=resolved_module_type,
config=resolved_module_config,
)
else:
raise Exception(f"Can't resolve module type: {module_type}")
else:
manifest = kiara.create_manifest(
module_or_operation=module_type, config=module_config
)
resolved_module_type = module_type
resolved_module_config = module_config
module = kiara.create_module(manifest=manifest)
step_id = step.get("step_id", None)
if not step_id:
if not auto_step_ids:
raise ValueError("Can't create step, no 'step_id' specified.")
else:
step_id = find_free_id(
slugify(manifest.module_type, separator="_"),
current_ids=step_ids,
)
if step_id in step_ids:
raise ValueError(
f"Can't create step: duplicate step id '{step_id}'."
)
step_ids.append(step_id)
input_links = {}
for input_field, sources in step.get("input_links", {}).items():
if isinstance(sources, str):
sources = [sources]
input_links[input_field] = sources
doc = step.get("doc", None)
# TODO: do we really need the deepcopy here?
_s = PipelineStep(
step_id=step_id,
module_type=resolved_module_type,
module_config=dict(resolved_module_config),
input_links=input_links, # type: ignore
doc=doc,
module_details=KiaraModuleInstance.from_module(module=module),
)
_s._module = module
result.append(_s)
else:
result.append(step)
return result
ensure_dict(v)
classmethod
¶Source code in kiara/models/module/pipeline/__init__.py
@validator("module_config", pre=True)
def ensure_dict(cls, v):
if v is None:
v = {}
return v
ensure_input_links_valid(v)
classmethod
¶Source code in kiara/models/module/pipeline/__init__.py
@validator("input_links", pre=True)
def ensure_input_links_valid(cls, v):
if v is None:
v = {}
result = {}
for input_name, output in v.items():
input_links = ensure_step_value_addresses(
default_field_name=input_name, link=output
)
result[input_name] = input_links
return result
ensure_valid_id(v)
classmethod
¶Source code in kiara/models/module/pipeline/__init__.py
@validator("step_id")
def ensure_valid_id(cls, v):
# TODO: check with regex
if "." in v or " " in v:
raise ValueError(
f"Step id can't contain special characters or whitespaces: {v}"
)
return v
validate_doc(value)
classmethod
¶Source code in kiara/models/module/pipeline/__init__.py
@validator("doc", pre=True)
def validate_doc(cls, value):
doc = DocumentationMetadataModel.create(value)
return doc
StepStatus (Enum)
¶Enum to describe the state of a workflow.
Source code in kiara/models/module/pipeline/__init__.py
class StepStatus(Enum):
"""Enum to describe the state of a workflow."""
INPUTS_INVALID = "inputs_invalid"
INPUTS_READY = "inputs_ready"
RESULTS_READY = "results_ready"
create_input_alias_map(steps)
¶Source code in kiara/models/module/pipeline/__init__.py
def create_input_alias_map(steps: Iterable[PipelineStep]) -> Dict[str, str]:
aliases: Dict[str, str] = {}
for step in steps:
field_names = step.module.input_names
for field_name in field_names:
alias = generate_pipeline_endpoint_name(
step_id=step.step_id, value_name=field_name
)
assert alias not in aliases.keys()
aliases[f"{step.step_id}.{field_name}"] = alias
return aliases
create_output_alias_map(steps)
¶Source code in kiara/models/module/pipeline/__init__.py
def create_output_alias_map(steps: Iterable[PipelineStep]) -> Dict[str, str]:
aliases: Dict[str, str] = {}
for step in steps:
field_names = step.module.output_names
for field_name in field_names:
alias = generate_pipeline_endpoint_name(
step_id=step.step_id, value_name=field_name
)
assert alias not in aliases.keys()
aliases[f"{step.step_id}.{field_name}"] = alias
return aliases
generate_pipeline_endpoint_name(step_id, value_name)
¶Source code in kiara/models/module/pipeline/__init__.py
def generate_pipeline_endpoint_name(step_id: str, value_name: str):
return f"{step_id}__{value_name}"
controller
¶logger
¶
PipelineController (PipelineListener)
¶Source code in kiara/models/module/pipeline/controller.py
class PipelineController(PipelineListener):
pass
SinglePipelineBatchController (SinglePipelineController)
¶A [PipelineController][kiara.models.modules.pipeline.controller.PipelineController] that executes all pipeline steps non-interactively.
This is the default implementation of a PipelineController, and probably the most simple implementation of one.
It waits until all inputs are set, after which it executes all pipeline steps in the required order.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline |
Pipeline |
the pipeline to control |
required |
auto_process |
bool |
whether to automatically start processing the pipeline as soon as the input set is valid |
True |
Source code in kiara/models/module/pipeline/controller.py
class SinglePipelineBatchController(SinglePipelineController):
"""A [PipelineController][kiara.models.modules.pipeline.controller.PipelineController] that executes all pipeline steps non-interactively.
This is the default implementation of a ``PipelineController``, and probably the most simple implementation of one.
It waits until all inputs are set, after which it executes all pipeline steps in the required order.
Arguments:
pipeline: the pipeline to control
auto_process: whether to automatically start processing the pipeline as soon as the input set is valid
"""
def __init__(
self,
pipeline: Pipeline,
job_registry: JobRegistry,
auto_process: bool = True,
):
self._auto_process: bool = auto_process
self._is_running: bool = False
super().__init__(pipeline=pipeline, job_registry=job_registry)
@property
def auto_process(self) -> bool:
return self._auto_process
@auto_process.setter
def auto_process(self, auto_process: bool):
self._auto_process = auto_process
def process_pipeline(self):
log = logger.bind(pipeline_id=self.pipeline.pipeline_id)
if self._is_running:
log.debug(
"ignore.pipeline_process",
reason="Pipeline already running.",
)
raise Exception("Pipeline already running.")
log.debug("execute.pipeline")
self._is_running = True
try:
for idx, stage in enumerate(
self.pipeline.structure.processing_stages, start=1
):
log.debug(
"execute.pipeline.stage",
stage=idx,
)
job_ids = {}
for step_id in stage:
log.debug(
"execute.pipeline.step",
step_id=step_id,
)
try:
job_id = self.process_step(step_id)
job_ids[step_id] = job_id
except Exception as e:
# TODO: cancel running jobs?
log_exception(e)
log.error(
"error.processing.pipeline",
step_id=step_id,
error=e,
)
return False
self.set_processing_results(job_ids=job_ids)
log.debug(
"execute_finished.pipeline.stage",
stage=idx,
)
finally:
self._is_running = False
log.debug("execute_finished.pipeline")
auto_process: bool
property
writable
¶process_pipeline(self)
¶Source code in kiara/models/module/pipeline/controller.py
def process_pipeline(self):
log = logger.bind(pipeline_id=self.pipeline.pipeline_id)
if self._is_running:
log.debug(
"ignore.pipeline_process",
reason="Pipeline already running.",
)
raise Exception("Pipeline already running.")
log.debug("execute.pipeline")
self._is_running = True
try:
for idx, stage in enumerate(
self.pipeline.structure.processing_stages, start=1
):
log.debug(
"execute.pipeline.stage",
stage=idx,
)
job_ids = {}
for step_id in stage:
log.debug(
"execute.pipeline.step",
step_id=step_id,
)
try:
job_id = self.process_step(step_id)
job_ids[step_id] = job_id
except Exception as e:
# TODO: cancel running jobs?
log_exception(e)
log.error(
"error.processing.pipeline",
step_id=step_id,
error=e,
)
return False
self.set_processing_results(job_ids=job_ids)
log.debug(
"execute_finished.pipeline.stage",
stage=idx,
)
finally:
self._is_running = False
log.debug("execute_finished.pipeline")
SinglePipelineController (PipelineController)
¶Source code in kiara/models/module/pipeline/controller.py
class SinglePipelineController(PipelineController):
def __init__(
self, job_registry: JobRegistry, pipeline: Union[Pipeline, None] = None
):
self._pipeline: Union[Pipeline, None] = None
self._job_registry: JobRegistry = job_registry
self._pipeline_details: Union[PipelineDetails, None] = None
if pipeline is not None:
self.pipeline = pipeline
@property
def pipeline(self) -> Pipeline:
if self._pipeline is None:
raise Exception("Pipeline not set (yet).")
return self._pipeline
@pipeline.setter
def pipeline(self, pipeline: Pipeline):
if self._pipeline is not None:
# TODO: destroy object?
self._pipeline._listeners.clear()
self._pipeline = pipeline
if self._pipeline is not None:
self._pipeline.add_listener(self)
def current_pipeline_state(self) -> PipelineDetails:
if self._pipeline_details is None:
self._pipeline_details = self.pipeline.get_pipeline_details()
return self._pipeline_details
def can_be_processed(self, step_id: str) -> bool:
"""Check whether the step with the provided id is ready to be processed."""
pipeline_state = self.current_pipeline_state()
step_state = pipeline_state.step_states[step_id]
return not step_state.invalid_details
def can_be_skipped(self, step_id: str) -> bool:
"""Check whether the processing of a step can be skipped."""
required = self.pipeline.structure.step_is_required(step_id=step_id)
if required:
required = self.can_be_processed(step_id)
return required
def _pipeline_event_occurred(self, event: PipelineEvent):
if event.pipeline_id != self.pipeline.pipeline_id:
return
self._pipeline_details = None
def set_processing_results(
self, job_ids: Mapping[str, uuid.UUID]
) -> Mapping[uuid.UUID, uuid.UUID]:
"""Set the processing results as values of the approrpiate step outputs.
Returns:
a dict with the result value id as key, and the id of the job that produced it as value
"""
self._job_registry.wait_for(*job_ids.values())
result: Dict[uuid.UUID, uuid.UUID] = {}
combined_outputs = {}
for step_id, job_id in job_ids.items():
record = self._job_registry.get_job_record(job_id=job_id)
if record is None:
continue
combined_outputs[step_id] = record.outputs
for output_id in record.outputs.values():
assert output_id not in result.keys()
result[output_id] = job_id
self.pipeline.set_multiple_step_outputs(
changed_outputs=combined_outputs, notify_listeners=True
)
return result
def pipeline_is_ready(self) -> bool:
"""Return whether the pipeline is ready to be processed.
A ``True`` result means that all pipeline inputs are set with valid values, and therefore every step within the
pipeline can be processed.
Returns:
whether the pipeline can be processed as a whole (``True``) or not (``False``)
"""
pipeline_inputs = self.pipeline._all_values.get_alias("pipeline.inputs")
assert pipeline_inputs is not None
return pipeline_inputs.all_items_valid
def process_step(self, step_id: str, wait: bool = False) -> uuid.UUID:
"""Kick off processing for the step with the provided id.
Arguments:
step_id: the id of the step that should be started
"""
job_config = self.pipeline.create_job_config_for_step(step_id)
job_metadata = {"is_pipeline_step": True, "step_id": step_id}
job_id = self._job_registry.execute_job(
job_config=job_config, job_metadata=job_metadata
)
# job_id = self._processor.create_job(job_config=job_config)
# self._processor.queue_job(job_id=job_id)
if wait:
self._job_registry.wait_for(job_id)
return job_id
pipeline: Pipeline
property
writable
¶can_be_processed(self, step_id)
¶Check whether the step with the provided id is ready to be processed.
Source code in kiara/models/module/pipeline/controller.py
def can_be_processed(self, step_id: str) -> bool:
"""Check whether the step with the provided id is ready to be processed."""
pipeline_state = self.current_pipeline_state()
step_state = pipeline_state.step_states[step_id]
return not step_state.invalid_details
can_be_skipped(self, step_id)
¶Check whether the processing of a step can be skipped.
Source code in kiara/models/module/pipeline/controller.py
def can_be_skipped(self, step_id: str) -> bool:
"""Check whether the processing of a step can be skipped."""
required = self.pipeline.structure.step_is_required(step_id=step_id)
if required:
required = self.can_be_processed(step_id)
return required
current_pipeline_state(self)
¶Source code in kiara/models/module/pipeline/controller.py
def current_pipeline_state(self) -> PipelineDetails:
if self._pipeline_details is None:
self._pipeline_details = self.pipeline.get_pipeline_details()
return self._pipeline_details
pipeline_is_ready(self)
¶Return whether the pipeline is ready to be processed.
A True result means that all pipeline inputs are set with valid values, and therefore every step within the
pipeline can be processed.
Returns:
| Type | Description |
|---|---|
bool |
whether the pipeline can be processed as a whole ( |
Source code in kiara/models/module/pipeline/controller.py
def pipeline_is_ready(self) -> bool:
"""Return whether the pipeline is ready to be processed.
A ``True`` result means that all pipeline inputs are set with valid values, and therefore every step within the
pipeline can be processed.
Returns:
whether the pipeline can be processed as a whole (``True``) or not (``False``)
"""
pipeline_inputs = self.pipeline._all_values.get_alias("pipeline.inputs")
assert pipeline_inputs is not None
return pipeline_inputs.all_items_valid
process_step(self, step_id, wait=False)
¶Kick off processing for the step with the provided id.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
str |
the id of the step that should be started |
required |
Source code in kiara/models/module/pipeline/controller.py
def process_step(self, step_id: str, wait: bool = False) -> uuid.UUID:
"""Kick off processing for the step with the provided id.
Arguments:
step_id: the id of the step that should be started
"""
job_config = self.pipeline.create_job_config_for_step(step_id)
job_metadata = {"is_pipeline_step": True, "step_id": step_id}
job_id = self._job_registry.execute_job(
job_config=job_config, job_metadata=job_metadata
)
# job_id = self._processor.create_job(job_config=job_config)
# self._processor.queue_job(job_id=job_id)
if wait:
self._job_registry.wait_for(job_id)
return job_id
set_processing_results(self, job_ids)
¶Set the processing results as values of the approrpiate step outputs.
Returns:
| Type | Description |
|---|---|
Mapping[uuid.UUID, uuid.UUID] |
a dict with the result value id as key, and the id of the job that produced it as value |
Source code in kiara/models/module/pipeline/controller.py
def set_processing_results(
self, job_ids: Mapping[str, uuid.UUID]
) -> Mapping[uuid.UUID, uuid.UUID]:
"""Set the processing results as values of the approrpiate step outputs.
Returns:
a dict with the result value id as key, and the id of the job that produced it as value
"""
self._job_registry.wait_for(*job_ids.values())
result: Dict[uuid.UUID, uuid.UUID] = {}
combined_outputs = {}
for step_id, job_id in job_ids.items():
record = self._job_registry.get_job_record(job_id=job_id)
if record is None:
continue
combined_outputs[step_id] = record.outputs
for output_id in record.outputs.values():
assert output_id not in result.keys()
result[output_id] = job_id
self.pipeline.set_multiple_step_outputs(
changed_outputs=combined_outputs, notify_listeners=True
)
return result
pipeline
¶yaml
¶
Pipeline
¶An instance of a [PipelineStructure][kiara.pipeline.structure.PipelineStructure] that holds state for all of the inputs/outputs of the steps within.
Source code in kiara/models/module/pipeline/pipeline.py
class Pipeline(object):
"""An instance of a [PipelineStructure][kiara.pipeline.structure.PipelineStructure] that holds state for all of the inputs/outputs of the steps within."""
@classmethod
def create_pipeline(
cls,
kiara: "Kiara",
pipeline: Union[PipelineConfig, PipelineStructure, Mapping, str],
) -> "Pipeline":
if isinstance(pipeline, Mapping):
pipeline_structure: PipelineStructure = PipelineConfig.from_config(
pipeline_name="__pipeline__", data=pipeline, kiara=kiara
).structure
elif isinstance(pipeline, PipelineConfig):
pipeline_structure = pipeline.structure
elif isinstance(pipeline, PipelineStructure):
pipeline_structure = pipeline
elif isinstance(pipeline, str):
operation = create_operation(module_or_operation=pipeline, kiara=kiara)
module = operation.module
if isinstance(module.config, PipelineConfig):
config: PipelineConfig = module.config
else:
raise NotImplementedError()
pipeline_structure = config.structure
else:
raise Exception(f"Invalid type for argument 'pipeline': {type(pipeline)}")
pipeline_obj = Pipeline(kiara=kiara, structure=pipeline_structure)
return pipeline_obj
def __init__(self, structure: PipelineStructure, kiara: "Kiara"):
self._id: uuid.UUID = uuid.uuid4()
self._structure: PipelineStructure = structure
self._value_refs: Mapping[AliasValueMap, Iterable[ValueRef]] = None # type: ignore
# self._status: StepStatus = StepStatus.STALE
self._steps_by_stage: Dict[int, Dict[str, PipelineStep]] = None # type: ignore
self._inputs_by_stage: Dict[int, List[str]] = None # type: ignore
self._outputs_by_stage: Dict[int, List[str]] = None # type: ignore
self._kiara: Kiara = kiara
self._data_registry: DataRegistry = kiara.data_registry
self._all_values: AliasValueMap = None # type: ignore
self._listeners: List[PipelineListener] = []
self._init_values()
# self._update_status()
@property
def pipeline_id(self) -> uuid.UUID:
return self._id
@property
def pipeline_name(self) -> str:
return self.structure.pipeline_config.pipeline_name
@property
def kiara_id(self) -> uuid.UUID:
return self._kiara.id
def _init_values(self):
"""Initialize this object. This should only be called once.
Basically, this goes through all the inputs and outputs of all steps, and 'allocates' a PipelineValueInfo object
for each of them. In case where output/input or pipeline-input/input points are connected, only one
value item is allocated, since those refer to the same value.
"""
values = AliasValueMap(
alias=str(self.id), version=0, assoc_value=None, values_schema={}
)
values._data_registry = self._data_registry
inputs_schema = self._structure.pipeline_inputs_schema
outputs_schema = self._structure.pipeline_outputs_schema
if inputs_schema:
for field_name, schema in inputs_schema.items():
values.set_alias_schema(f"pipeline.inputs.{field_name}", schema=schema)
else:
values.set_alias_schema("pipeline.inputs", schema=ValueSchema(type="none"))
if outputs_schema:
for field_name, schema in outputs_schema.items():
values.set_alias_schema(f"pipeline.outputs.{field_name}", schema=schema)
else:
values.set_alias_schema("pipeline.outputs", schema=ValueSchema(type="none"))
for step_id in self.step_ids:
step = self.get_step(step_id)
for field_name, value_schema in step.module.inputs_schema.items():
values.set_alias_schema(
f"steps.{step_id}.inputs.{field_name}", schema=value_schema
)
for field_name, value_schema in step.module.outputs_schema.items():
values.set_alias_schema(
f"steps.{step_id}.outputs.{field_name}", schema=value_schema
)
self._all_values = values
initial_inputs = {
k: SpecialValue.NOT_SET
for k in self._structure.pipeline_inputs_schema.keys()
}
self.set_pipeline_inputs(inputs=initial_inputs)
def __eq__(self, other):
if not isinstance(other, Pipeline):
return False
return self._id == other._id
def __hash__(self):
return hash(self._id)
def add_listener(self, listener: PipelineListener):
self._listeners.append(listener)
@property
def id(self) -> uuid.UUID:
return self._id
@property
def structure(self) -> PipelineStructure:
return self._structure
@property
def doc(self) -> DocumentationMetadataModel:
return self.structure.pipeline_config.doc
def get_current_pipeline_inputs(self) -> Dict[str, uuid.UUID]:
"""All (pipeline) input values of this pipeline."""
alias_map = self._all_values.get_alias("pipeline.inputs")
return alias_map.get_all_value_ids() # type: ignore
def get_current_pipeline_outputs(self) -> Dict[str, uuid.UUID]:
"""All (pipeline) output values of this pipeline."""
alias_map = self._all_values.get_alias("pipeline.outputs")
return alias_map.get_all_value_ids() # type: ignore
def get_current_step_inputs(self, step_id) -> Dict[str, uuid.UUID]:
alias_map = self._all_values.get_alias(f"steps.{step_id}.inputs")
return alias_map.get_all_value_ids() # type: ignore
def get_current_step_outputs(self, step_id) -> Dict[str, uuid.UUID]:
alias_map = self._all_values.get_alias(f"steps.{step_id}.outputs")
return alias_map.get_all_value_ids() # type: ignore
def get_inputs_for_steps(self, *step_ids: str) -> Dict[str, Dict[str, uuid.UUID]]:
"""Retrieve value ids for the inputs of the specified steps (or all steps, if no argument provided."""
result = {}
for step_id in self._structure.step_ids:
if step_ids and step_id not in step_ids:
continue
ids = self.get_current_step_inputs(step_id=step_id)
result[step_id] = ids
return result
def get_outputs_for_steps(self, *step_ids: str) -> Dict[str, Dict[str, uuid.UUID]]:
"""Retrieve value ids for the outputs of the specified steps (or all steps, if no argument provided."""
result = {}
for step_id in self._structure.step_ids:
if step_ids and step_id not in step_ids:
continue
ids = self.get_current_step_outputs(step_id=step_id)
result[step_id] = ids
return result
def _notify_pipeline_listeners(self, event: PipelineEvent):
for listener in self._listeners:
listener._pipeline_event_occurred(event=event)
def get_pipeline_details(self) -> PipelineDetails:
pipeline_inputs = self._all_values.get_alias("pipeline.inputs")
pipeline_outputs = self._all_values.get_alias("pipeline.outputs")
if pipeline_inputs:
invalid = pipeline_inputs.check_invalid()
if not invalid:
status = StepStatus.INPUTS_READY
step_outputs = self._all_values.get_alias("pipeline.outputs")
assert step_outputs is not None
invalid_outputs = step_outputs.check_invalid()
# TODO: also check that all the pedigrees match up with current inputs
if not invalid_outputs:
status = StepStatus.RESULTS_READY
else:
status = StepStatus.INPUTS_INVALID
_pipeline_inputs = pipeline_inputs.get_all_value_ids()
else:
_pipeline_inputs = {}
invalid = {}
status = StepStatus.INPUTS_READY
if pipeline_outputs:
_pipeline_outputs = pipeline_outputs.get_all_value_ids()
else:
_pipeline_outputs = {}
step_states = {}
for step_id in self._structure.step_ids:
d = self.get_step_details(step_id)
step_states[step_id] = d
details = PipelineDetails.construct(
kiara_id=self._data_registry.kiara_id,
pipeline_id=self.pipeline_id,
pipeline_status=status,
pipeline_inputs=_pipeline_inputs,
pipeline_outputs=_pipeline_outputs,
invalid_details=invalid,
step_states=step_states,
)
return details
def get_step_details(self, step_id: str) -> StepDetails:
step_input_ids = self.get_current_step_inputs(step_id=step_id)
step_output_ids = self.get_current_step_outputs(step_id=step_id)
step_inputs = self._all_values.get_alias(f"steps.{step_id}.inputs")
assert step_inputs is not None
invalid = step_inputs.check_invalid()
processing_stage = self._structure.get_processing_stage(step_id)
if not invalid:
status = StepStatus.INPUTS_READY
step_outputs = self._all_values.get_alias(f"steps.{step_id}.outputs")
assert step_outputs is not None
invalid_outputs = step_outputs.check_invalid()
# TODO: also check that all the pedigrees match up with current inputs
if not invalid_outputs:
status = StepStatus.RESULTS_READY
else:
status = StepStatus.INPUTS_INVALID
details = StepDetails.construct(
kiara_id=self._data_registry.kiara_id,
pipeline_id=self.pipeline_id,
step=self._structure.get_step(step_id=step_id),
step_id=step_id,
status=status,
inputs=step_input_ids,
outputs=step_output_ids,
invalid_details=invalid,
processing_stage=processing_stage,
)
return details
def set_pipeline_inputs(
self,
inputs: Mapping[str, Any],
sync_to_step_inputs: bool = True,
notify_listeners: bool = True,
) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
values_to_set: Dict[str, uuid.UUID] = {}
for k, v in inputs.items():
if v is SpecialValue.NOT_SET:
values_to_set[k] = NOT_SET_VALUE_ID
elif v in [None, SpecialValue.NO_VALUE]:
values_to_set[k] = NONE_VALUE_ID
else:
alias_map = self._all_values.get_alias("pipeline.inputs")
assert alias_map is not None
# dbg(alias_map.__dict__)
schema = alias_map.values_schema.get(k, None)
if schema is None:
raise Exception(
f"Can't set pipeline input for input '{k}': no such input field. Available fields: {', '.join(alias_map.values_schema.keys())}"
)
value = self._data_registry.register_data(
data=v, schema=schema, pedigree=ORPHAN, reuse_existing=True
)
values_to_set[k] = value.value_id
if not values_to_set:
return {}
changed_pipeline_inputs = self._set_values("pipeline.inputs", **values_to_set)
changed_results = {"__pipeline__": {"inputs": changed_pipeline_inputs}}
if sync_to_step_inputs:
changed = self.sync_pipeline_inputs(notify_listeners=False)
dpath.util.merge(changed_results, changed) # type: ignore
if notify_listeners:
event = PipelineEvent.create_event(pipeline=self, changed=changed_results)
if event:
self._notify_pipeline_listeners(event)
return changed_results
def sync_pipeline_inputs(
self, notify_listeners: bool = True
) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
pipeline_inputs = self.get_current_pipeline_inputs()
values_to_sync: Dict[str, Dict[str, Union[uuid.UUID, None]]] = {}
for field_name, ref in self._structure.pipeline_input_refs.items():
for step_input in ref.connected_inputs:
step_inputs = self.get_current_step_inputs(step_input.step_id)
if step_inputs[step_input.value_name] != pipeline_inputs[field_name]:
values_to_sync.setdefault(step_input.step_id, {})[
step_input.value_name
] = pipeline_inputs[field_name]
results: Dict[str, Mapping[str, Mapping[str, ChangedValue]]] = {}
for step_id in values_to_sync.keys():
values = values_to_sync[step_id]
step_changed = self._set_step_inputs(step_id=step_id, inputs=values)
dpath.util.merge(results, step_changed) # type: ignore
if notify_listeners:
event = PipelineEvent.create_event(pipeline=self, changed=results)
if event:
self._notify_pipeline_listeners(event)
return results
def _set_step_inputs(
self, step_id: str, inputs: Mapping[str, Union[uuid.UUID, None]]
) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
changed_step_inputs = self._set_values(f"steps.{step_id}.inputs", **inputs)
if not changed_step_inputs:
return {}
result: Dict[str, Dict[str, Dict[str, ChangedValue]]] = {
step_id: {"inputs": changed_step_inputs}
}
step_outputs = self._structure.get_step_output_refs(step_id=step_id)
null_outputs = {k: NOT_SET_VALUE_ID for k in step_outputs.keys()}
changed_outputs = self.set_step_outputs(
step_id=step_id, outputs=null_outputs, notify_listeners=False
)
# assert step_id in changed_outputs.keys()
result.update(changed_outputs) # type: ignore
return result
def set_multiple_step_outputs(
self,
changed_outputs: Mapping[str, Mapping[str, Union[uuid.UUID, None]]],
notify_listeners: bool = True,
) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
results: Dict[str, Dict[str, Dict[str, ChangedValue]]] = {}
for step_id, outputs in changed_outputs.items():
step_results = self.set_step_outputs(
step_id=step_id, outputs=outputs, notify_listeners=False
)
dpath.util.merge(results, step_results) # type: ignore
if notify_listeners:
event = PipelineEvent.create_event(pipeline=self, changed=results)
if event:
self._notify_pipeline_listeners(event)
return results
def set_step_outputs(
self,
step_id: str,
outputs: Mapping[str, Union[uuid.UUID, None]],
notify_listeners: bool = True,
) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
# make sure pedigrees match with respective inputs?
changed_step_outputs = self._set_values(f"steps.{step_id}.outputs", **outputs)
if not changed_step_outputs:
return {}
result: Dict[str, Dict[str, Dict[str, ChangedValue]]] = {
step_id: {"outputs": changed_step_outputs}
}
output_refs = self._structure.get_step_output_refs(step_id=step_id)
pipeline_outputs: Dict[str, Union[uuid.UUID, None]] = {}
inputs_to_set: Dict[str, Dict[str, Union[uuid.UUID, None]]] = {}
for field_name, ref in output_refs.items():
if ref.pipeline_output:
assert ref.pipeline_output not in pipeline_outputs.keys()
pipeline_outputs[ref.pipeline_output] = outputs[field_name]
for input_ref in ref.connected_inputs:
inputs_to_set.setdefault(input_ref.step_id, {})[
input_ref.value_name
] = outputs[field_name]
for step_id, step_inputs in inputs_to_set.items():
changed_step_fields = self._set_step_inputs(
step_id=step_id, inputs=step_inputs
)
dpath.util.merge(result, changed_step_fields) # type: ignore
if pipeline_outputs:
changed_pipeline_outputs = self._set_pipeline_outputs(**pipeline_outputs)
dpath.util.merge( # type: ignore
result, {"__pipeline__": {"outputs": changed_pipeline_outputs}}
)
if notify_listeners:
event = PipelineEvent.create_event(pipeline=self, changed=result)
if event:
self._notify_pipeline_listeners(event)
return result
def _set_pipeline_outputs(
self, **outputs: Union[uuid.UUID, None]
) -> Mapping[str, ChangedValue]:
changed_pipeline_outputs = self._set_values("pipeline.outputs", **outputs)
return changed_pipeline_outputs
def _set_values(
self, alias: str, **values: Union[uuid.UUID, None]
) -> Dict[str, ChangedValue]:
"""Set values (value-ids) for the sub-alias-map with the specified alias path."""
invalid = {}
for k in values.keys():
_alias = self._all_values.get_alias(alias)
assert _alias is not None
if k not in _alias.values_schema.keys():
invalid[
k
] = f"Invalid field '{k}'. Available fields: {', '.join(self.get_current_pipeline_inputs().keys())}"
if invalid:
raise InvalidValuesException(invalid_values=invalid)
alias_map: Union[AliasValueMap, None] = self._all_values.get_alias(alias)
assert alias_map is not None
values_to_set: Dict[str, Union[uuid.UUID, None]] = {}
current: Dict[str, Union[uuid.UUID, None]] = {}
changed: Dict[str, ChangedValue] = {}
for field_name, new_value in values.items():
current_value = self._all_values.get_alias(f"{alias}.{field_name}")
if current_value is not None:
current_value_id = current_value.assoc_value
else:
current_value_id = None
current[field_name] = current_value_id
if current_value_id != new_value:
values_to_set[field_name] = new_value
changed[field_name] = ChangedValue(old=current_value_id, new=new_value)
_alias = self._all_values.get_alias(alias)
assert _alias is not None
_alias._set_aliases(**values_to_set)
return changed
@property
def step_ids(self) -> Iterable[str]:
"""Return all ids of the steps of this pipeline."""
return self._structure.step_ids
@property
def execution_graph(self) -> nx.DiGraph:
return self._structure.execution_graph
@property
def data_flow_graph(self) -> nx.DiGraph:
return self._structure.data_flow_graph
@property
def data_flow_graph_simple(self) -> nx.DiGraph:
return self._structure.data_flow_graph_simple
def get_step(self, step_id: str) -> PipelineStep:
"""Return the object representing a step in this workflow, identified by the step id."""
return self._structure.get_step(step_id)
def get_steps_by_stage(
self,
) -> Mapping[int, Mapping[str, PipelineStep]]:
"""Return a all pipeline steps, ordered by stage they belong to."""
if self._steps_by_stage is not None:
return self._steps_by_stage
result: Dict[int, Dict[str, PipelineStep]] = {}
for step_id in self.step_ids:
step = self.get_step(step_id)
stage = self._structure.get_processing_stage(step.step_id)
assert stage is not None
result.setdefault(stage, {})[step_id] = step
self._steps_by_stage = result
return self._steps_by_stage
def create_job_config_for_step(self, step_id: str) -> JobConfig:
step_inputs: Mapping[str, uuid.UUID] = self.get_current_step_inputs(step_id)
step_details: StepDetails = self.get_step_details(step_id=step_id)
step: PipelineStep = self.get_step(step_id=step_id)
# if the inputs are not valid, ignore this step
if step_details.status == StepStatus.INPUTS_INVALID:
invalid_details = step_details.invalid_details
assert invalid_details is not None
msg = f"Can't execute step '{step_id}', invalid inputs: {', '.join(invalid_details.keys())}"
raise InvalidValuesException(msg=msg, invalid_values=invalid_details)
job_config = JobConfig.create_from_module(
data_registry=self._data_registry, module=step.module, inputs=step_inputs
)
return job_config
def create_renderable(self, **config: Any) -> RenderableType:
return PipelineInfo.create_from_pipeline(
kiara=self._kiara, pipeline=self
).create_renderable(**config)
data_flow_graph: DiGraph
property
readonly
¶data_flow_graph_simple: DiGraph
property
readonly
¶doc: DocumentationMetadataModel
property
readonly
¶execution_graph: DiGraph
property
readonly
¶id: UUID
property
readonly
¶kiara_id: UUID
property
readonly
¶pipeline_id: UUID
property
readonly
¶pipeline_name: str
property
readonly
¶step_ids: Iterable[str]
property
readonly
¶Return all ids of the steps of this pipeline.
structure: PipelineStructure
property
readonly
¶add_listener(self, listener)
¶Source code in kiara/models/module/pipeline/pipeline.py
def add_listener(self, listener: PipelineListener):
self._listeners.append(listener)
create_job_config_for_step(self, step_id)
¶Source code in kiara/models/module/pipeline/pipeline.py
def create_job_config_for_step(self, step_id: str) -> JobConfig:
step_inputs: Mapping[str, uuid.UUID] = self.get_current_step_inputs(step_id)
step_details: StepDetails = self.get_step_details(step_id=step_id)
step: PipelineStep = self.get_step(step_id=step_id)
# if the inputs are not valid, ignore this step
if step_details.status == StepStatus.INPUTS_INVALID:
invalid_details = step_details.invalid_details
assert invalid_details is not None
msg = f"Can't execute step '{step_id}', invalid inputs: {', '.join(invalid_details.keys())}"
raise InvalidValuesException(msg=msg, invalid_values=invalid_details)
job_config = JobConfig.create_from_module(
data_registry=self._data_registry, module=step.module, inputs=step_inputs
)
return job_config
create_pipeline(kiara, pipeline)
classmethod
¶Source code in kiara/models/module/pipeline/pipeline.py
@classmethod
def create_pipeline(
cls,
kiara: "Kiara",
pipeline: Union[PipelineConfig, PipelineStructure, Mapping, str],
) -> "Pipeline":
if isinstance(pipeline, Mapping):
pipeline_structure: PipelineStructure = PipelineConfig.from_config(
pipeline_name="__pipeline__", data=pipeline, kiara=kiara
).structure
elif isinstance(pipeline, PipelineConfig):
pipeline_structure = pipeline.structure
elif isinstance(pipeline, PipelineStructure):
pipeline_structure = pipeline
elif isinstance(pipeline, str):
operation = create_operation(module_or_operation=pipeline, kiara=kiara)
module = operation.module
if isinstance(module.config, PipelineConfig):
config: PipelineConfig = module.config
else:
raise NotImplementedError()
pipeline_structure = config.structure
else:
raise Exception(f"Invalid type for argument 'pipeline': {type(pipeline)}")
pipeline_obj = Pipeline(kiara=kiara, structure=pipeline_structure)
return pipeline_obj
create_renderable(self, **config)
¶Source code in kiara/models/module/pipeline/pipeline.py
def create_renderable(self, **config: Any) -> RenderableType:
return PipelineInfo.create_from_pipeline(
kiara=self._kiara, pipeline=self
).create_renderable(**config)
get_current_pipeline_inputs(self)
¶All (pipeline) input values of this pipeline.
Source code in kiara/models/module/pipeline/pipeline.py
def get_current_pipeline_inputs(self) -> Dict[str, uuid.UUID]:
"""All (pipeline) input values of this pipeline."""
alias_map = self._all_values.get_alias("pipeline.inputs")
return alias_map.get_all_value_ids() # type: ignore
get_current_pipeline_outputs(self)
¶All (pipeline) output values of this pipeline.
Source code in kiara/models/module/pipeline/pipeline.py
def get_current_pipeline_outputs(self) -> Dict[str, uuid.UUID]:
"""All (pipeline) output values of this pipeline."""
alias_map = self._all_values.get_alias("pipeline.outputs")
return alias_map.get_all_value_ids() # type: ignore
get_current_step_inputs(self, step_id)
¶Source code in kiara/models/module/pipeline/pipeline.py
def get_current_step_inputs(self, step_id) -> Dict[str, uuid.UUID]:
alias_map = self._all_values.get_alias(f"steps.{step_id}.inputs")
return alias_map.get_all_value_ids() # type: ignore
get_current_step_outputs(self, step_id)
¶Source code in kiara/models/module/pipeline/pipeline.py
def get_current_step_outputs(self, step_id) -> Dict[str, uuid.UUID]:
alias_map = self._all_values.get_alias(f"steps.{step_id}.outputs")
return alias_map.get_all_value_ids() # type: ignore
get_inputs_for_steps(self, *step_ids)
¶Retrieve value ids for the inputs of the specified steps (or all steps, if no argument provided.
Source code in kiara/models/module/pipeline/pipeline.py
def get_inputs_for_steps(self, *step_ids: str) -> Dict[str, Dict[str, uuid.UUID]]:
"""Retrieve value ids for the inputs of the specified steps (or all steps, if no argument provided."""
result = {}
for step_id in self._structure.step_ids:
if step_ids and step_id not in step_ids:
continue
ids = self.get_current_step_inputs(step_id=step_id)
result[step_id] = ids
return result
get_outputs_for_steps(self, *step_ids)
¶Retrieve value ids for the outputs of the specified steps (or all steps, if no argument provided.
Source code in kiara/models/module/pipeline/pipeline.py
def get_outputs_for_steps(self, *step_ids: str) -> Dict[str, Dict[str, uuid.UUID]]:
"""Retrieve value ids for the outputs of the specified steps (or all steps, if no argument provided."""
result = {}
for step_id in self._structure.step_ids:
if step_ids and step_id not in step_ids:
continue
ids = self.get_current_step_outputs(step_id=step_id)
result[step_id] = ids
return result
get_pipeline_details(self)
¶Source code in kiara/models/module/pipeline/pipeline.py
def get_pipeline_details(self) -> PipelineDetails:
pipeline_inputs = self._all_values.get_alias("pipeline.inputs")
pipeline_outputs = self._all_values.get_alias("pipeline.outputs")
if pipeline_inputs:
invalid = pipeline_inputs.check_invalid()
if not invalid:
status = StepStatus.INPUTS_READY
step_outputs = self._all_values.get_alias("pipeline.outputs")
assert step_outputs is not None
invalid_outputs = step_outputs.check_invalid()
# TODO: also check that all the pedigrees match up with current inputs
if not invalid_outputs:
status = StepStatus.RESULTS_READY
else:
status = StepStatus.INPUTS_INVALID
_pipeline_inputs = pipeline_inputs.get_all_value_ids()
else:
_pipeline_inputs = {}
invalid = {}
status = StepStatus.INPUTS_READY
if pipeline_outputs:
_pipeline_outputs = pipeline_outputs.get_all_value_ids()
else:
_pipeline_outputs = {}
step_states = {}
for step_id in self._structure.step_ids:
d = self.get_step_details(step_id)
step_states[step_id] = d
details = PipelineDetails.construct(
kiara_id=self._data_registry.kiara_id,
pipeline_id=self.pipeline_id,
pipeline_status=status,
pipeline_inputs=_pipeline_inputs,
pipeline_outputs=_pipeline_outputs,
invalid_details=invalid,
step_states=step_states,
)
return details
get_step(self, step_id)
¶Return the object representing a step in this workflow, identified by the step id.
Source code in kiara/models/module/pipeline/pipeline.py
def get_step(self, step_id: str) -> PipelineStep:
"""Return the object representing a step in this workflow, identified by the step id."""
return self._structure.get_step(step_id)
get_step_details(self, step_id)
¶Source code in kiara/models/module/pipeline/pipeline.py
def get_step_details(self, step_id: str) -> StepDetails:
step_input_ids = self.get_current_step_inputs(step_id=step_id)
step_output_ids = self.get_current_step_outputs(step_id=step_id)
step_inputs = self._all_values.get_alias(f"steps.{step_id}.inputs")
assert step_inputs is not None
invalid = step_inputs.check_invalid()
processing_stage = self._structure.get_processing_stage(step_id)
if not invalid:
status = StepStatus.INPUTS_READY
step_outputs = self._all_values.get_alias(f"steps.{step_id}.outputs")
assert step_outputs is not None
invalid_outputs = step_outputs.check_invalid()
# TODO: also check that all the pedigrees match up with current inputs
if not invalid_outputs:
status = StepStatus.RESULTS_READY
else:
status = StepStatus.INPUTS_INVALID
details = StepDetails.construct(
kiara_id=self._data_registry.kiara_id,
pipeline_id=self.pipeline_id,
step=self._structure.get_step(step_id=step_id),
step_id=step_id,
status=status,
inputs=step_input_ids,
outputs=step_output_ids,
invalid_details=invalid,
processing_stage=processing_stage,
)
return details
get_steps_by_stage(self)
¶Return a all pipeline steps, ordered by stage they belong to.
Source code in kiara/models/module/pipeline/pipeline.py
def get_steps_by_stage(
self,
) -> Mapping[int, Mapping[str, PipelineStep]]:
"""Return a all pipeline steps, ordered by stage they belong to."""
if self._steps_by_stage is not None:
return self._steps_by_stage
result: Dict[int, Dict[str, PipelineStep]] = {}
for step_id in self.step_ids:
step = self.get_step(step_id)
stage = self._structure.get_processing_stage(step.step_id)
assert stage is not None
result.setdefault(stage, {})[step_id] = step
self._steps_by_stage = result
return self._steps_by_stage
set_multiple_step_outputs(self, changed_outputs, notify_listeners=True)
¶Source code in kiara/models/module/pipeline/pipeline.py
def set_multiple_step_outputs(
self,
changed_outputs: Mapping[str, Mapping[str, Union[uuid.UUID, None]]],
notify_listeners: bool = True,
) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
results: Dict[str, Dict[str, Dict[str, ChangedValue]]] = {}
for step_id, outputs in changed_outputs.items():
step_results = self.set_step_outputs(
step_id=step_id, outputs=outputs, notify_listeners=False
)
dpath.util.merge(results, step_results) # type: ignore
if notify_listeners:
event = PipelineEvent.create_event(pipeline=self, changed=results)
if event:
self._notify_pipeline_listeners(event)
return results
set_pipeline_inputs(self, inputs, sync_to_step_inputs=True, notify_listeners=True)
¶Source code in kiara/models/module/pipeline/pipeline.py
def set_pipeline_inputs(
self,
inputs: Mapping[str, Any],
sync_to_step_inputs: bool = True,
notify_listeners: bool = True,
) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
values_to_set: Dict[str, uuid.UUID] = {}
for k, v in inputs.items():
if v is SpecialValue.NOT_SET:
values_to_set[k] = NOT_SET_VALUE_ID
elif v in [None, SpecialValue.NO_VALUE]:
values_to_set[k] = NONE_VALUE_ID
else:
alias_map = self._all_values.get_alias("pipeline.inputs")
assert alias_map is not None
# dbg(alias_map.__dict__)
schema = alias_map.values_schema.get(k, None)
if schema is None:
raise Exception(
f"Can't set pipeline input for input '{k}': no such input field. Available fields: {', '.join(alias_map.values_schema.keys())}"
)
value = self._data_registry.register_data(
data=v, schema=schema, pedigree=ORPHAN, reuse_existing=True
)
values_to_set[k] = value.value_id
if not values_to_set:
return {}
changed_pipeline_inputs = self._set_values("pipeline.inputs", **values_to_set)
changed_results = {"__pipeline__": {"inputs": changed_pipeline_inputs}}
if sync_to_step_inputs:
changed = self.sync_pipeline_inputs(notify_listeners=False)
dpath.util.merge(changed_results, changed) # type: ignore
if notify_listeners:
event = PipelineEvent.create_event(pipeline=self, changed=changed_results)
if event:
self._notify_pipeline_listeners(event)
return changed_results
set_step_outputs(self, step_id, outputs, notify_listeners=True)
¶Source code in kiara/models/module/pipeline/pipeline.py
def set_step_outputs(
self,
step_id: str,
outputs: Mapping[str, Union[uuid.UUID, None]],
notify_listeners: bool = True,
) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
# make sure pedigrees match with respective inputs?
changed_step_outputs = self._set_values(f"steps.{step_id}.outputs", **outputs)
if not changed_step_outputs:
return {}
result: Dict[str, Dict[str, Dict[str, ChangedValue]]] = {
step_id: {"outputs": changed_step_outputs}
}
output_refs = self._structure.get_step_output_refs(step_id=step_id)
pipeline_outputs: Dict[str, Union[uuid.UUID, None]] = {}
inputs_to_set: Dict[str, Dict[str, Union[uuid.UUID, None]]] = {}
for field_name, ref in output_refs.items():
if ref.pipeline_output:
assert ref.pipeline_output not in pipeline_outputs.keys()
pipeline_outputs[ref.pipeline_output] = outputs[field_name]
for input_ref in ref.connected_inputs:
inputs_to_set.setdefault(input_ref.step_id, {})[
input_ref.value_name
] = outputs[field_name]
for step_id, step_inputs in inputs_to_set.items():
changed_step_fields = self._set_step_inputs(
step_id=step_id, inputs=step_inputs
)
dpath.util.merge(result, changed_step_fields) # type: ignore
if pipeline_outputs:
changed_pipeline_outputs = self._set_pipeline_outputs(**pipeline_outputs)
dpath.util.merge( # type: ignore
result, {"__pipeline__": {"outputs": changed_pipeline_outputs}}
)
if notify_listeners:
event = PipelineEvent.create_event(pipeline=self, changed=result)
if event:
self._notify_pipeline_listeners(event)
return result
sync_pipeline_inputs(self, notify_listeners=True)
¶Source code in kiara/models/module/pipeline/pipeline.py
def sync_pipeline_inputs(
self, notify_listeners: bool = True
) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]]]:
pipeline_inputs = self.get_current_pipeline_inputs()
values_to_sync: Dict[str, Dict[str, Union[uuid.UUID, None]]] = {}
for field_name, ref in self._structure.pipeline_input_refs.items():
for step_input in ref.connected_inputs:
step_inputs = self.get_current_step_inputs(step_input.step_id)
if step_inputs[step_input.value_name] != pipeline_inputs[field_name]:
values_to_sync.setdefault(step_input.step_id, {})[
step_input.value_name
] = pipeline_inputs[field_name]
results: Dict[str, Mapping[str, Mapping[str, ChangedValue]]] = {}
for step_id in values_to_sync.keys():
values = values_to_sync[step_id]
step_changed = self._set_step_inputs(step_id=step_id, inputs=values)
dpath.util.merge(results, step_changed) # type: ignore
if notify_listeners:
event = PipelineEvent.create_event(pipeline=self, changed=results)
if event:
self._notify_pipeline_listeners(event)
return results
PipelineInfo (ItemInfo)
pydantic-model
¶Source code in kiara/models/module/pipeline/pipeline.py
class PipelineInfo(ItemInfo[Pipeline]):
_kiara_model_id = "info.pipeline"
@classmethod
def base_instance_class(cls) -> Type[Pipeline]:
return Pipeline
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: Pipeline, **kwargs):
return cls.create_from_pipeline(kiara=kiara, pipeline=instance)
@classmethod
def category_name(cls) -> str:
return "pipeline"
@classmethod
def create_from_pipeline(cls, kiara: "Kiara", pipeline: Pipeline):
doc = DocumentationMetadataModel.create(None)
authors = AuthorsMetadataModel()
context = ContextMetadataModel()
pipeline_info = PipelineInfo(
type_name=str(pipeline.pipeline_id),
documentation=doc,
authors=authors,
context=context,
pipeline_structure=pipeline.structure,
pipeline_details=pipeline.get_pipeline_details(),
)
pipeline_info._kiara = kiara
return pipeline_info
pipeline_structure: PipelineStructure = Field(description="The pipeline structure.")
pipeline_details: PipelineDetails = Field(description="The current input details.")
_kiara: "Kiara" = PrivateAttr(default=None)
def create_pipeline_table(self, **config: Any) -> Table:
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
self._fill_table(table=table, config=config)
return table
def _fill_table(self, table: Table, config: Mapping[str, Any]):
include_pipeline_inputs = config.get("include_pipeline_inputs", True)
include_pipeline_outputs = config.get("include_pipeline_outputs", True)
include_steps = config.get("include_steps", True)
if include_pipeline_inputs:
input_values = self._kiara.data_registry.create_valuemap(
data=self.pipeline_details.pipeline_inputs,
schema=self.pipeline_structure.pipeline_inputs_schema,
)
ordered_fields: Dict[str, List[str]] = {}
for field_name, ref in self.pipeline_structure.pipeline_input_refs.items():
for con_input in ref.connected_inputs:
details = self.pipeline_structure.get_step_details(
step_id=con_input.step_id
)
stage = details["processing_stage"]
ordered_fields.setdefault(stage, []).append(field_name)
fields = []
for stage in sorted(ordered_fields.keys()):
for f in sorted(ordered_fields[stage]):
if f not in fields:
fields.append(f)
inputs = create_value_map_status_renderable(
input_values,
render_config={
"show_description": False,
"show_type": False,
"show_default": True,
"show_value_ids": True,
},
fields=fields,
)
table.add_row("pipeline inputs", inputs)
if include_steps:
steps = create_pipeline_steps_tree(
pipeline_structure=self.pipeline_structure,
pipeline_details=self.pipeline_details,
)
table.add_row("steps", steps)
if include_pipeline_outputs:
output_values = self._kiara.data_registry.load_values(
values=self.pipeline_details.pipeline_outputs
)
ordered_fields = {}
for (
field_name,
o_ref,
) in self.pipeline_structure.pipeline_output_refs.items():
con_step_id = o_ref.connected_output.step_id
details = self.pipeline_structure.get_step_details(step_id=con_step_id)
stage = details["processing_stage"]
ordered_fields.setdefault(stage, []).append(field_name)
fields = []
for stage in sorted(ordered_fields.keys()):
for f in sorted(ordered_fields[stage]):
fields.append(f)
t_outputs = create_value_map_status_renderable(
output_values,
render_config={
"show_description": False,
"show_type": True,
"show_default": False,
"show_required": False,
"show_value_ids": True,
},
fields=fields,
)
table.add_row("pipeline outputs", t_outputs)
def create_renderable(self, **config: Any) -> RenderableType:
include_details = config.get("include_details", False)
include_doc = config.get("include_doc", False)
include_authors = config.get("include_authors", False)
include_context = config.get("include_context", False)
include_structure = config.get("include_structure", False)
table = self.create_pipeline_table(**config)
if include_details:
t_details = create_table_from_model_object(
self.pipeline_details, render_config=config
)
table.add_row("details", t_details)
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
if include_authors:
table.add_row("Author(s)", self.authors.create_renderable(**config))
if include_context:
table.add_row("Context", self.context.create_renderable(**config))
if include_structure:
table.add_row(
"Pipeline structure",
self.pipeline_structure.create_renderable(**config),
)
return table
pipeline_details: PipelineDetails
pydantic-field
required
¶The current input details.
pipeline_structure: PipelineStructure
pydantic-field
required
¶The pipeline structure.
base_instance_class()
classmethod
¶Source code in kiara/models/module/pipeline/pipeline.py
@classmethod
def base_instance_class(cls) -> Type[Pipeline]:
return Pipeline
category_name()
classmethod
¶Source code in kiara/models/module/pipeline/pipeline.py
@classmethod
def category_name(cls) -> str:
return "pipeline"
create_from_instance(kiara, instance, **kwargs)
classmethod
¶Source code in kiara/models/module/pipeline/pipeline.py
@classmethod
def create_from_instance(cls, kiara: "Kiara", instance: Pipeline, **kwargs):
return cls.create_from_pipeline(kiara=kiara, pipeline=instance)
create_from_pipeline(kiara, pipeline)
classmethod
¶Source code in kiara/models/module/pipeline/pipeline.py
@classmethod
def create_from_pipeline(cls, kiara: "Kiara", pipeline: Pipeline):
doc = DocumentationMetadataModel.create(None)
authors = AuthorsMetadataModel()
context = ContextMetadataModel()
pipeline_info = PipelineInfo(
type_name=str(pipeline.pipeline_id),
documentation=doc,
authors=authors,
context=context,
pipeline_structure=pipeline.structure,
pipeline_details=pipeline.get_pipeline_details(),
)
pipeline_info._kiara = kiara
return pipeline_info
create_pipeline_table(self, **config)
¶Source code in kiara/models/module/pipeline/pipeline.py
def create_pipeline_table(self, **config: Any) -> Table:
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
self._fill_table(table=table, config=config)
return table
create_renderable(self, **config)
¶Source code in kiara/models/module/pipeline/pipeline.py
def create_renderable(self, **config: Any) -> RenderableType:
include_details = config.get("include_details", False)
include_doc = config.get("include_doc", False)
include_authors = config.get("include_authors", False)
include_context = config.get("include_context", False)
include_structure = config.get("include_structure", False)
table = self.create_pipeline_table(**config)
if include_details:
t_details = create_table_from_model_object(
self.pipeline_details, render_config=config
)
table.add_row("details", t_details)
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
if include_authors:
table.add_row("Author(s)", self.authors.create_renderable(**config))
if include_context:
table.add_row("Context", self.context.create_renderable(**config))
if include_structure:
table.add_row(
"Pipeline structure",
self.pipeline_structure.create_renderable(**config),
)
return table
PipelineListener (ABC)
¶Source code in kiara/models/module/pipeline/pipeline.py
class PipelineListener(abc.ABC):
@abc.abstractmethod
def _pipeline_event_occurred(self, event: PipelineEvent):
pass
structure
¶
PipelineStructure (KiaraModel)
pydantic-model
¶An object that holds one or several steps, and describes the connections between them.
Source code in kiara/models/module/pipeline/structure.py
class PipelineStructure(KiaraModel):
"""An object that holds one or several steps, and describes the connections between them."""
_kiara_model_id = "instance.pipeline_structure"
pipeline_config: PipelineConfig = Field(
description="The underlying pipeline config."
)
steps: List[PipelineStep] = Field(description="The pipeline steps ")
input_aliases: Dict[str, str] = Field(description="The input aliases.")
output_aliases: Dict[str, str] = Field(description="The output aliases.")
@root_validator(pre=True)
def validate_pipeline_config(cls, values):
pipeline_config = values.get("pipeline_config", None)
if not pipeline_config:
raise ValueError("No 'pipeline_config' provided.")
if len(values) != 1:
raise ValueError(
"Only 'pipeline_config' key allowed when creating a pipeline structure object."
)
if isinstance(pipeline_config, Mapping):
pipeline_config = PipelineConfig(**pipeline_config)
_config: PipelineConfig = pipeline_config
_steps: List[PipelineStep] = list(_config.steps)
_input_aliases: Dict[str, str] = dict(_config.input_aliases)
_output_aliases: Dict[str, str] = dict(_config.output_aliases)
invalid_input_aliases = [a for a in _input_aliases.values() if "." in a]
if invalid_input_aliases:
raise Exception(
f"Invalid input aliases, aliases can't contain special characters: {', '.join(invalid_input_aliases)}."
)
invalid_output_aliases = [a for a in _input_aliases.values() if "." in a]
if invalid_input_aliases:
raise Exception(
f"Invalid input aliases, aliases can't contain special characters: {', '.join(invalid_output_aliases)}."
)
valid_input_names = set()
for step in _steps:
for input_name in step.module.input_names:
valid_input_names.add(f"{step.step_id}.{input_name}")
invalid_input_aliases = [
a for a in _input_aliases.keys() if a not in valid_input_names
]
if invalid_input_aliases:
raise Exception(
f"Invalid input reference(s): {', '.join(invalid_input_aliases)}. Must be one of: {', '.join(valid_input_names)}"
)
valid_output_names = set()
for step in _steps:
for output_name in step.module.output_names:
valid_output_names.add(f"{step.step_id}.{output_name}")
invalid_output_names = [
a for a in _output_aliases.keys() if a not in valid_output_names
]
if invalid_output_names:
raise Exception(
f"Invalid output reference(s): {', '.join(invalid_output_names)}. Must be one of: {', '.join(valid_output_names)}"
)
values["steps"] = _steps
values["input_aliases"] = _input_aliases
values["output_aliases"] = _output_aliases
return values
# this is hardcoded for now
_add_all_workflow_outputs: bool = PrivateAttr(default=False)
_constants: Dict[str, Any] = PrivateAttr(default=None) # type: ignore
_defaults: Dict[str, Any] = PrivateAttr(None) # type: ignore
_execution_graph: nx.DiGraph = PrivateAttr(None) # type: ignore
_data_flow_graph: nx.DiGraph = PrivateAttr(None) # type: ignore
_data_flow_graph_simple: nx.DiGraph = PrivateAttr(None) # type: ignore
_processing_stages: List[List[str]] = PrivateAttr(None) # type: ignore
# holds details about the (current) processing steps contained in this workflow
_steps_details: Dict[str, Any] = PrivateAttr(None) # type: ignore
def _retrieve_data_to_hash(self) -> Any:
return {
"steps": [step.instance_cid for step in self.steps],
"input_aliases": self.input_aliases,
"output_aliases": self.output_aliases,
}
def _retrieve_id(self) -> str:
return self.pipeline_config.instance_id
@property
def steps_details(self) -> Mapping[str, Any]:
if self._steps_details is None:
self._process_steps()
return self._steps_details # type: ignore
@property
def step_ids(self) -> Iterable[str]:
if self._steps_details is None:
self._process_steps()
return self._steps_details.keys() # type: ignore
@property
def constants(self) -> Mapping[str, Any]:
if self._constants is None:
self._process_steps()
return self._constants # type: ignore
@property
def defaults(self) -> Mapping[str, Any]:
if self._defaults is None:
self._process_steps()
return self._defaults # type: ignore
def get_step(self, step_id: str) -> PipelineStep:
d = self.steps_details.get(step_id, None)
if d is None:
raise Exception(f"No step with id: {step_id}")
return d["step"]
def get_step_input_refs(self, step_id: str) -> Mapping[str, StepInputRef]:
d = self.steps_details.get(step_id, None)
if d is None:
raise Exception(f"No step with id: {step_id}")
return d["inputs"]
def get_step_output_refs(self, step_id: str) -> Mapping[str, StepOutputRef]:
d = self.steps_details.get(step_id, None)
if d is None:
raise Exception(f"No step with id: {step_id}")
return d["outputs"]
def get_step_details(self, step_id: str) -> Mapping[str, Any]:
d = self.steps_details.get(step_id, None)
if d is None:
raise Exception(f"No step with id: {step_id}")
return d
@property
def execution_graph(self) -> nx.DiGraph:
if self._execution_graph is None:
self._process_steps()
return self._execution_graph
@property
def data_flow_graph(self) -> nx.DiGraph:
if self._data_flow_graph is None:
self._process_steps()
return self._data_flow_graph
@property
def data_flow_graph_simple(self) -> nx.DiGraph:
if self._data_flow_graph_simple is None:
self._process_steps()
return self._data_flow_graph_simple
@property
def processing_stages(self) -> List[List[str]]:
if self._steps_details is None:
self._process_steps()
return self._processing_stages
@lru_cache()
def _get_node_of_type(self, node_type: str):
if self._steps_details is None:
self._process_steps()
return [
node
for node, attr in self._data_flow_graph.nodes(data=True)
if attr["type"] == node_type
]
@property
def steps_input_refs(self) -> Dict[str, StepInputRef]:
return {
node.alias: node
for node in self._get_node_of_type(node_type=StepInputRef.__name__)
}
@property
def steps_output_refs(self) -> Dict[str, StepOutputRef]:
return {
node.alias: node
for node in self._get_node_of_type(node_type=StepOutputRef.__name__)
}
@property
def pipeline_input_refs(self) -> Dict[str, PipelineInputRef]:
return {
node.value_name: node
for node in self._get_node_of_type(node_type=PipelineInputRef.__name__)
}
@property
def pipeline_output_refs(self) -> Dict[str, PipelineOutputRef]:
return {
node.value_name: node
for node in self._get_node_of_type(node_type=PipelineOutputRef.__name__)
}
@property
def pipeline_inputs_schema(self) -> Mapping[str, ValueSchema]:
schemas = {
input_name: w_in.value_schema
for input_name, w_in in self.pipeline_input_refs.items()
}
return schemas
@property
def pipeline_outputs_schema(self) -> Mapping[str, ValueSchema]:
return {
output_name: w_out.value_schema
for output_name, w_out in self.pipeline_output_refs.items()
}
def get_processing_stage(self, step_id: str) -> int:
"""Return the processing stage for the specified step_id.
Returns the stage nr (starting with '1').
"""
for index, stage in enumerate(self.processing_stages, start=1):
if step_id in stage:
return index
raise Exception(f"Invalid step id '{step_id}'.")
def step_is_required(self, step_id: str) -> bool:
"""Check if the specified step is required, or can be omitted."""
return self.get_step_details(step_id=step_id)["required"]
def _process_steps(self):
"""The core method of this class, it connects all the processing modules, their inputs and outputs."""
steps_details: Dict[str, Any] = {}
execution_graph = nx.DiGraph()
execution_graph.add_node("__root__")
data_flow_graph = nx.DiGraph()
data_flow_graph_simple = nx.DiGraph()
processing_stages = []
constants = {}
structure_defaults = {}
# temp variable, to hold all outputs
outputs: Dict[str, StepOutputRef] = {}
# process all pipeline and step outputs first
_temp_steps_map: Dict[str, PipelineStep] = {}
pipeline_outputs: Dict[str, PipelineOutputRef] = {}
for step in self.steps:
_temp_steps_map[step.step_id] = step
if step.step_id in steps_details.keys():
raise Exception(
f"Can't process steps: duplicate step_id '{step.step_id}'"
)
steps_details[step.step_id] = {
"step": step,
"outputs": {},
"inputs": {},
"required": True,
}
data_flow_graph.add_node(step, type="step")
# go through all the module outputs, create points for them and connect them to pipeline outputs
for output_name, schema in step.module.outputs_schema.items():
step_output = StepOutputRef(
value_name=output_name,
value_schema=schema,
step_id=step.step_id,
)
steps_details[step.step_id]["outputs"][output_name] = step_output
step_alias = generate_step_alias(step.step_id, output_name)
outputs[step_alias] = step_output
# step_output_name = generate_pipeline_endpoint_name(
# step_id=step.step_id, value_name=output_name
# )
step_output_name = f"{step.step_id}.{output_name}"
if not self.output_aliases:
raise NotImplementedError()
if step_output_name in self.output_aliases.keys():
step_output_name = self.output_aliases[step_output_name]
else:
if not self._add_all_workflow_outputs:
# this output is not interesting for the workflow
step_output_name = None
if step_output_name:
step_output_address = StepValueAddress(
step_id=step.step_id, value_name=output_name
)
pipeline_output = PipelineOutputRef(
value_name=step_output_name,
connected_output=step_output_address,
value_schema=schema,
)
pipeline_outputs[step_output_name] = pipeline_output
step_output.pipeline_output = pipeline_output.value_name
data_flow_graph.add_node(
pipeline_output, type=PipelineOutputRef.__name__
)
data_flow_graph.add_edge(step_output, pipeline_output)
data_flow_graph_simple.add_node(
pipeline_output, type=PipelineOutputRef.__name__
)
data_flow_graph_simple.add_edge(step, pipeline_output)
data_flow_graph.add_node(step_output, type=StepOutputRef.__name__)
data_flow_graph.add_edge(step, step_output)
# now process inputs, and connect them to the appropriate output/pipeline-input points
existing_pipeline_input_points: Dict[str, PipelineInputRef] = {}
for step in self.steps:
other_step_dependency: Set = set()
# go through all the inputs of a module, create input points and connect them to either
# other module outputs, or pipeline inputs (which need to be created)
module_constants: Mapping[str, Any] = step.module.get_config_value(
"constants"
)
for input_name, schema in step.module.inputs_schema.items():
matching_input_links: List[StepValueAddress] = []
is_constant = input_name in module_constants.keys()
for value_name, input_links in step.input_links.items():
if value_name == input_name:
for input_link in input_links:
if input_link in matching_input_links:
raise Exception(f"Duplicate input link: {input_link}")
matching_input_links.append(input_link)
if matching_input_links:
# this means we connect to other steps output
connected_output_points: List[StepOutputRef] = []
connected_outputs: List[StepValueAddress] = []
for input_link in matching_input_links:
output_id = generate_step_alias(
input_link.step_id, input_link.value_name
)
if output_id not in outputs.keys():
raise Exception(
f"Can't connect input '{input_name}' for step '{step.step_id}': no output '{output_id}' available. Available output names: {', '.join(outputs.keys())}"
)
connected_output_points.append(outputs[output_id])
connected_outputs.append(input_link)
other_step_dependency.add(input_link.step_id)
step_input_point = StepInputRef(
step_id=step.step_id,
value_name=input_name,
value_schema=schema,
is_constant=is_constant,
connected_pipeline_input=None,
connected_outputs=connected_outputs,
)
for op in connected_output_points:
op.connected_inputs.append(step_input_point.address)
data_flow_graph.add_edge(op, step_input_point)
data_flow_graph_simple.add_edge(
_temp_steps_map[op.step_id], step_input_point
) # TODO: name edge
data_flow_graph_simple.add_edge(
step_input_point, step
) # TODO: name edge
else:
# this means we connect to pipeline input
# pipeline_input_name = generate_pipeline_endpoint_name(
# step_id=step.step_id, value_name=input_name
# )
pipeline_input_ref = f"{step.step_id}.{input_name}"
# check whether this input has an alias associated with it
if not self.input_aliases:
raise NotImplementedError()
if pipeline_input_ref in self.input_aliases.keys():
# this means we use the pipeline alias
pipeline_input_name = self.input_aliases[pipeline_input_ref]
else:
pipeline_input_name = generate_pipeline_endpoint_name(
step_id=step.step_id, value_name=input_name
)
if pipeline_input_name in existing_pipeline_input_points.keys():
# we already created a pipeline input with this name
# TODO: check whether schema fits
connected_pipeline_input = existing_pipeline_input_points[
pipeline_input_name
]
assert connected_pipeline_input.is_constant == is_constant
else:
# we need to create the pipeline input
connected_pipeline_input = PipelineInputRef(
value_name=pipeline_input_name,
value_schema=schema,
is_constant=is_constant,
)
existing_pipeline_input_points[
pipeline_input_name
] = connected_pipeline_input
data_flow_graph.add_node(
connected_pipeline_input, type=PipelineInputRef.__name__
)
data_flow_graph_simple.add_node(
connected_pipeline_input, type=PipelineInputRef.__name__
)
if is_constant:
constants[
pipeline_input_name
] = step.module.get_config_value("constants")[input_name]
default_val = step.module.get_config_value("defaults").get(
input_name, None
)
if is_constant and default_val is not None:
raise Exception(
f"Module config invalid for step '{step.step_id}': both default value and constant provided for input '{input_name}'."
)
elif default_val is not None:
structure_defaults[pipeline_input_name] = default_val
step_input_point = StepInputRef(
step_id=step.step_id,
value_name=input_name,
value_schema=schema,
connected_pipeline_input=connected_pipeline_input.value_name,
connected_outputs=None,
)
connected_pipeline_input.connected_inputs.append(
step_input_point.address
)
data_flow_graph.add_edge(connected_pipeline_input, step_input_point)
data_flow_graph_simple.add_edge(connected_pipeline_input, step)
data_flow_graph.add_node(step_input_point, type=StepInputRef.__name__)
steps_details[step.step_id]["inputs"][input_name] = step_input_point
data_flow_graph.add_edge(step_input_point, step)
if other_step_dependency:
for module_id in other_step_dependency:
execution_graph.add_edge(module_id, step.step_id)
else:
execution_graph.add_edge("__root__", step.step_id)
# calculate execution order
path_lengths: Dict[str, int] = {}
for step in self.steps:
step_id = step.step_id
paths = list(nx.all_simple_paths(execution_graph, "__root__", step_id))
max_steps = max(paths, key=lambda x: len(x))
path_lengths[step_id] = len(max_steps) - 1
if path_lengths.values():
max_length = max(path_lengths.values())
for i in range(1, max_length + 1):
stage: List[str] = [
m for m, length in path_lengths.items() if length == i
]
processing_stages.append(stage)
for _step_id in stage:
steps_details[_step_id]["processing_stage"] = i
# steps_details[_step_id]["step"].processing_stage = i
self._constants = constants
self._defaults = structure_defaults
self._steps_details = steps_details
self._execution_graph = execution_graph
self._data_flow_graph = data_flow_graph
self._data_flow_graph_simple = data_flow_graph_simple
self._processing_stages = processing_stages
self._get_node_of_type.cache_clear()
def create_renderable(self, **config: Any) -> RenderableType:
tree = Tree("pipeline")
inputs = tree.add("inputs")
for field_name, schema in self.pipeline_inputs_schema.items():
inputs.add(f"[i]{field_name}[i] (type: {schema.type})")
steps = tree.add("steps")
for idx, stage in enumerate(self.processing_stages, start=1):
stage_node = steps.add(f"stage {idx}")
for step_id in stage:
step_node = stage_node.add(f"step: {step_id}")
step = self.get_step(step_id=step_id)
if step.doc.is_set:
step_node.add(f"desc: {step.doc.description}")
step_node.add(f"module: {step.module_type}")
outputs = tree.add("outputs")
for field_name, schema in self.pipeline_outputs_schema.items():
outputs.add(f"[i]{field_name}[i] (type: {schema.type})")
return tree
constants: Mapping[str, Any]
property
readonly
¶data_flow_graph: DiGraph
property
readonly
¶data_flow_graph_simple: DiGraph
property
readonly
¶defaults: Mapping[str, Any]
property
readonly
¶execution_graph: DiGraph
property
readonly
¶input_aliases: Dict[str, str]
pydantic-field
required
¶The input aliases.
output_aliases: Dict[str, str]
pydantic-field
required
¶The output aliases.
pipeline_config: PipelineConfig
pydantic-field
required
¶The underlying pipeline config.
pipeline_input_refs: Dict[str, kiara.models.module.pipeline.value_refs.PipelineInputRef]
property
readonly
¶pipeline_inputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶pipeline_output_refs: Dict[str, kiara.models.module.pipeline.value_refs.PipelineOutputRef]
property
readonly
¶pipeline_outputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶processing_stages: List[List[str]]
property
readonly
¶step_ids: Iterable[str]
property
readonly
¶steps: List[kiara.models.module.pipeline.PipelineStep]
pydantic-field
required
¶The pipeline steps
steps_details: Mapping[str, Any]
property
readonly
¶steps_input_refs: Dict[str, kiara.models.module.pipeline.value_refs.StepInputRef]
property
readonly
¶steps_output_refs: Dict[str, kiara.models.module.pipeline.value_refs.StepOutputRef]
property
readonly
¶create_renderable(self, **config)
¶Source code in kiara/models/module/pipeline/structure.py
def create_renderable(self, **config: Any) -> RenderableType:
tree = Tree("pipeline")
inputs = tree.add("inputs")
for field_name, schema in self.pipeline_inputs_schema.items():
inputs.add(f"[i]{field_name}[i] (type: {schema.type})")
steps = tree.add("steps")
for idx, stage in enumerate(self.processing_stages, start=1):
stage_node = steps.add(f"stage {idx}")
for step_id in stage:
step_node = stage_node.add(f"step: {step_id}")
step = self.get_step(step_id=step_id)
if step.doc.is_set:
step_node.add(f"desc: {step.doc.description}")
step_node.add(f"module: {step.module_type}")
outputs = tree.add("outputs")
for field_name, schema in self.pipeline_outputs_schema.items():
outputs.add(f"[i]{field_name}[i] (type: {schema.type})")
return tree
get_processing_stage(self, step_id)
¶Return the processing stage for the specified step_id.
Returns the stage nr (starting with '1').
Source code in kiara/models/module/pipeline/structure.py
def get_processing_stage(self, step_id: str) -> int:
"""Return the processing stage for the specified step_id.
Returns the stage nr (starting with '1').
"""
for index, stage in enumerate(self.processing_stages, start=1):
if step_id in stage:
return index
raise Exception(f"Invalid step id '{step_id}'.")
get_step(self, step_id)
¶Source code in kiara/models/module/pipeline/structure.py
def get_step(self, step_id: str) -> PipelineStep:
d = self.steps_details.get(step_id, None)
if d is None:
raise Exception(f"No step with id: {step_id}")
return d["step"]
get_step_details(self, step_id)
¶Source code in kiara/models/module/pipeline/structure.py
def get_step_details(self, step_id: str) -> Mapping[str, Any]:
d = self.steps_details.get(step_id, None)
if d is None:
raise Exception(f"No step with id: {step_id}")
return d
get_step_input_refs(self, step_id)
¶Source code in kiara/models/module/pipeline/structure.py
def get_step_input_refs(self, step_id: str) -> Mapping[str, StepInputRef]:
d = self.steps_details.get(step_id, None)
if d is None:
raise Exception(f"No step with id: {step_id}")
return d["inputs"]
get_step_output_refs(self, step_id)
¶Source code in kiara/models/module/pipeline/structure.py
def get_step_output_refs(self, step_id: str) -> Mapping[str, StepOutputRef]:
d = self.steps_details.get(step_id, None)
if d is None:
raise Exception(f"No step with id: {step_id}")
return d["outputs"]
step_is_required(self, step_id)
¶Check if the specified step is required, or can be omitted.
Source code in kiara/models/module/pipeline/structure.py
def step_is_required(self, step_id: str) -> bool:
"""Check if the specified step is required, or can be omitted."""
return self.get_step_details(step_id=step_id)["required"]
validate_pipeline_config(values)
classmethod
¶Source code in kiara/models/module/pipeline/structure.py
@root_validator(pre=True)
def validate_pipeline_config(cls, values):
pipeline_config = values.get("pipeline_config", None)
if not pipeline_config:
raise ValueError("No 'pipeline_config' provided.")
if len(values) != 1:
raise ValueError(
"Only 'pipeline_config' key allowed when creating a pipeline structure object."
)
if isinstance(pipeline_config, Mapping):
pipeline_config = PipelineConfig(**pipeline_config)
_config: PipelineConfig = pipeline_config
_steps: List[PipelineStep] = list(_config.steps)
_input_aliases: Dict[str, str] = dict(_config.input_aliases)
_output_aliases: Dict[str, str] = dict(_config.output_aliases)
invalid_input_aliases = [a for a in _input_aliases.values() if "." in a]
if invalid_input_aliases:
raise Exception(
f"Invalid input aliases, aliases can't contain special characters: {', '.join(invalid_input_aliases)}."
)
invalid_output_aliases = [a for a in _input_aliases.values() if "." in a]
if invalid_input_aliases:
raise Exception(
f"Invalid input aliases, aliases can't contain special characters: {', '.join(invalid_output_aliases)}."
)
valid_input_names = set()
for step in _steps:
for input_name in step.module.input_names:
valid_input_names.add(f"{step.step_id}.{input_name}")
invalid_input_aliases = [
a for a in _input_aliases.keys() if a not in valid_input_names
]
if invalid_input_aliases:
raise Exception(
f"Invalid input reference(s): {', '.join(invalid_input_aliases)}. Must be one of: {', '.join(valid_input_names)}"
)
valid_output_names = set()
for step in _steps:
for output_name in step.module.output_names:
valid_output_names.add(f"{step.step_id}.{output_name}")
invalid_output_names = [
a for a in _output_aliases.keys() if a not in valid_output_names
]
if invalid_output_names:
raise Exception(
f"Invalid output reference(s): {', '.join(invalid_output_names)}. Must be one of: {', '.join(valid_output_names)}"
)
values["steps"] = _steps
values["input_aliases"] = _input_aliases
values["output_aliases"] = _output_aliases
return values
generate_pipeline_endpoint_name(step_id, value_name)
¶Source code in kiara/models/module/pipeline/structure.py
def generate_pipeline_endpoint_name(step_id: str, value_name: str):
return f"{step_id}__{value_name}"
value_refs
¶
PipelineInputRef (ValueRef)
pydantic-model
¶An input to a pipeline.
Source code in kiara/models/module/pipeline/value_refs.py
class PipelineInputRef(ValueRef):
"""An input to a pipeline."""
connected_inputs: List[StepValueAddress] = Field(
description="The step inputs that are connected to this pipeline input",
default_factory=list,
)
is_constant: bool = Field(
"Whether this input is a constant and can't be changed by the user."
)
@property
def alias(self) -> str:
return generate_step_alias(PIPELINE_PARENT_MARKER, self.value_name)
PipelineOutputRef (ValueRef)
pydantic-model
¶An output to a pipeline.
Source code in kiara/models/module/pipeline/value_refs.py
class PipelineOutputRef(ValueRef):
"""An output to a pipeline."""
connected_output: StepValueAddress = Field(description="Connected step outputs.")
@property
def alias(self) -> str:
return generate_step_alias(PIPELINE_PARENT_MARKER, self.value_name)
StepInputRef (ValueRef)
pydantic-model
¶An input to a step.
This object can either have a 'connected_outputs' set, or a 'connected_pipeline_input', not both.
Source code in kiara/models/module/pipeline/value_refs.py
class StepInputRef(ValueRef):
"""An input to a step.
This object can either have a 'connected_outputs' set, or a 'connected_pipeline_input', not both.
"""
step_id: str = Field(description="The step id.")
connected_outputs: Union[List[StepValueAddress], None] = Field(
default=None,
description="A potential connected list of one or several module outputs.",
)
connected_pipeline_input: Union[str, None] = Field(
default=None, description="A potential pipeline input."
)
is_constant: bool = Field(
"Whether this input is a constant and can't be changed by the user."
)
@root_validator(pre=True)
def ensure_single_connected_item(cls, values):
if values.get("connected_outputs", None) and values.get(
"connected_pipeline_input"
):
raise ValueError("Multiple connected items, only one allowed.")
return values
@property
def alias(self) -> str:
return generate_step_alias(self.step_id, self.value_name)
@property
def address(self) -> StepValueAddress:
return StepValueAddress(step_id=self.step_id, value_name=self.value_name)
def __str__(self):
name = camel_case_to_snake_case(self.__class__.__name__[0:-5], repl=" ")
return f"{name}: {self.step_id}.{self.value_name} ({self.value_schema.type})"
address: StepValueAddress
property
readonly
¶alias: str
property
readonly
¶connected_outputs: List[kiara.models.module.pipeline.value_refs.StepValueAddress]
pydantic-field
¶A potential connected list of one or several module outputs.
connected_pipeline_input: str
pydantic-field
¶A potential pipeline input.
is_constant: bool
pydantic-field
¶step_id: str
pydantic-field
required
¶The step id.
ensure_single_connected_item(values)
classmethod
¶Source code in kiara/models/module/pipeline/value_refs.py
@root_validator(pre=True)
def ensure_single_connected_item(cls, values):
if values.get("connected_outputs", None) and values.get(
"connected_pipeline_input"
):
raise ValueError("Multiple connected items, only one allowed.")
return values
StepOutputRef (ValueRef)
pydantic-model
¶An output to a step.
Source code in kiara/models/module/pipeline/value_refs.py
class StepOutputRef(ValueRef):
"""An output to a step."""
class Config:
allow_mutation = True
step_id: str = Field(description="The step id.")
pipeline_output: Union[str, None] = Field(
description="The connected pipeline output."
)
connected_inputs: List[StepValueAddress] = Field(
description="The step inputs that are connected to this step output",
default_factory=list,
)
@property
def alias(self) -> str:
return generate_step_alias(self.step_id, self.value_name)
@property
def address(self) -> StepValueAddress:
return StepValueAddress(step_id=self.step_id, value_name=self.value_name)
def __str__(self):
name = camel_case_to_snake_case(self.__class__.__name__[0:-5], repl=" ")
return f"{name}: {self.step_id}.{self.value_name} ({self.value_schema.type})"
address: StepValueAddress
property
readonly
¶alias: str
property
readonly
¶connected_inputs: List[kiara.models.module.pipeline.value_refs.StepValueAddress]
pydantic-field
¶The step inputs that are connected to this step output
pipeline_output: str
pydantic-field
¶The connected pipeline output.
step_id: str
pydantic-field
required
¶The step id.
Config
¶Source code in kiara/models/module/pipeline/value_refs.py
class Config:
allow_mutation = True
StepValueAddress (BaseModel)
pydantic-model
¶Small model to describe the address of a value of a step, within a Pipeline/PipelineStructure.
Source code in kiara/models/module/pipeline/value_refs.py
class StepValueAddress(BaseModel):
"""Small model to describe the address of a value of a step, within a Pipeline/PipelineStructure."""
class Config:
extra = Extra.forbid
step_id: str = Field(description="The id of a step within a pipeline.")
value_name: str = Field(
description="The name of the value (output name or pipeline input name)."
)
sub_value: Union[Dict[str, Any], None] = Field(
default=None,
description="A reference to a subitem of a value (e.g. column, list item)",
)
@property
def alias(self):
"""An alias string for this address (in the form ``[step_id].[value_name]``)."""
return generate_step_alias(self.step_id, self.value_name)
def __eq__(self, other):
if not isinstance(other, StepValueAddress):
return False
return (self.step_id, self.value_name, self.sub_value) == (
other.step_id,
other.value_name,
other.sub_value,
)
def __hash__(self):
return hash((self.step_id, self.value_name, self.sub_value))
def __repr__(self):
if self.sub_value:
sub_value = f" sub_value={self.sub_value}"
else:
sub_value = ""
return f"{self.__class__.__name__}(step_id={self.step_id}, value_name={self.value_name}{sub_value})"
def __str__(self):
return self.__repr__()
alias
property
readonly
¶An alias string for this address (in the form [step_id].[value_name]).
step_id: str
pydantic-field
required
¶The id of a step within a pipeline.
sub_value: Dict[str, Any]
pydantic-field
¶A reference to a subitem of a value (e.g. column, list item)
value_name: str
pydantic-field
required
¶The name of the value (output name or pipeline input name).
Config
¶Source code in kiara/models/module/pipeline/value_refs.py
class Config:
extra = Extra.forbid
ValueRef (BaseModel)
pydantic-model
¶An object that holds information about the location of a value within a pipeline (or other structure).
Basically, a ValueRef helps the containing object where in its structure the value belongs (for example so
it can update dependent other values). A ValueRef object (obviously) does not contain the value itself.
There are four different ValueRef type that are relevant for pipelines:
- [kiara.pipeline.values.StepInputRef][]: an input to a step
- [kiara.pipeline.values.StepOutputRef][]: an output of a step
- [kiara.pipeline.values.PipelineInputRef][]: an input to a pipeline
- [kiara.pipeline.values.PipelineOutputRef][]: an output for a pipeline
Several ValueRef objects can target the same value, for example a step output and a connected step input would
reference the same Value (in most cases)..
Source code in kiara/models/module/pipeline/value_refs.py
class ValueRef(BaseModel):
"""An object that holds information about the location of a value within a pipeline (or other structure).
Basically, a `ValueRef` helps the containing object where in its structure the value belongs (for example so
it can update dependent other values). A `ValueRef` object (obviously) does not contain the value itself.
There are four different ValueRef type that are relevant for pipelines:
- [kiara.pipeline.values.StepInputRef][]: an input to a step
- [kiara.pipeline.values.StepOutputRef][]: an output of a step
- [kiara.pipeline.values.PipelineInputRef][]: an input to a pipeline
- [kiara.pipeline.values.PipelineOutputRef][]: an output for a pipeline
Several `ValueRef` objects can target the same value, for example a step output and a connected step input would
reference the same `Value` (in most cases)..
"""
class Config:
allow_mutation = True
extra = Extra.forbid
_id: uuid.UUID = PrivateAttr(default_factory=uuid.uuid4)
value_name: str
value_schema: ValueSchema
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
return self._id == other._id
def __hash__(self):
return hash(self._id)
def __repr__(self):
step_id = ""
if hasattr(self, "step_id"):
step_id = f" step_id='{self.step_id}'"
return f"{self.__class__.__name__}(value_name='{self.value_name}' {step_id})"
def __str__(self):
name = camel_case_to_snake_case(self.__class__.__name__[0:-5], repl=" ")
return f"{name}: {self.value_name} ({self.value_schema.type})"
generate_step_alias(step_id, value_name)
¶Source code in kiara/models/module/pipeline/value_refs.py
def generate_step_alias(step_id: str, value_name):
return f"{step_id}.{value_name}"
python_class
¶
Classes¶
KiaraModuleInstance (PythonClass)
pydantic-model
¶Source code in kiara/models/python_class.py
class KiaraModuleInstance(PythonClass):
_kiara_model_id: str = "metadata.kiara_module_class"
@classmethod
def from_module(cls, module: "KiaraModule"):
item_cls = module.__class__
cls_name = item_cls.__name__
module_name = item_cls.__module__
if module_name == "builtins":
full_name = cls_name
else:
full_name = f"{item_cls.__module__}.{item_cls.__name__}"
conf: Dict[str, Any] = {
"python_class_name": cls_name,
"python_module_name": module_name,
"full_name": full_name,
}
conf["module_config"] = module.config
conf["inputs_schema"] = module.inputs_schema
conf["outputs_schema"] = module.outputs_schema
result = KiaraModuleInstance.construct(**conf)
result._cls_cache = item_cls
result._module_instance_cache = module
return result
module_config: Dict[str, Any] = Field(description="The module config.")
inputs_schema: Dict[str, ValueSchema] = Field(
description="The schema for the module input(s)."
)
outputs_schema: Dict[str, ValueSchema] = Field(
description="The schema for the module output(s)."
)
_module_instance_cache: "KiaraModule" = PrivateAttr(default=None)
def get_kiara_module_instance(self) -> "KiaraModule":
if self._module_instance_cache is not None:
return self._module_instance_cache
m_cls = self.get_class()
self._module_instance_cache = m_cls(module_config=self.module_config)
return self._module_instance_cache
inputs_schema: Dict[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The schema for the module input(s).
module_config: Dict[str, Any]
pydantic-field
required
¶The module config.
outputs_schema: Dict[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The schema for the module output(s).
from_module(module)
classmethod
¶Source code in kiara/models/python_class.py
@classmethod
def from_module(cls, module: "KiaraModule"):
item_cls = module.__class__
cls_name = item_cls.__name__
module_name = item_cls.__module__
if module_name == "builtins":
full_name = cls_name
else:
full_name = f"{item_cls.__module__}.{item_cls.__name__}"
conf: Dict[str, Any] = {
"python_class_name": cls_name,
"python_module_name": module_name,
"full_name": full_name,
}
conf["module_config"] = module.config
conf["inputs_schema"] = module.inputs_schema
conf["outputs_schema"] = module.outputs_schema
result = KiaraModuleInstance.construct(**conf)
result._cls_cache = item_cls
result._module_instance_cache = module
return result
get_kiara_module_instance(self)
¶Source code in kiara/models/python_class.py
def get_kiara_module_instance(self) -> "KiaraModule":
if self._module_instance_cache is not None:
return self._module_instance_cache
m_cls = self.get_class()
self._module_instance_cache = m_cls(module_config=self.module_config)
return self._module_instance_cache
PythonClass (KiaraModel)
pydantic-model
¶Python class and module information.
Source code in kiara/models/python_class.py
class PythonClass(KiaraModel):
"""Python class and module information."""
_kiara_model_id = "instance.wrapped_python_class"
@classmethod
def from_class(cls, item_cls: Type, attach_context_metadata: bool = False):
cls_name = item_cls.__name__
module_name = item_cls.__module__
if module_name == "builtins":
full_name = cls_name
else:
full_name = f"{item_cls.__module__}.{item_cls.__name__}"
conf: Dict[str, Any] = {
"python_class_name": cls_name,
"python_module_name": module_name,
"full_name": full_name,
}
if attach_context_metadata:
raise NotImplementedError()
ctx_md = ContextMetadataModel.from_class(item_cls=item_cls)
conf["items"] = ctx_md
result = PythonClass.construct(**conf)
result._cls_cache = item_cls
return result
python_class_name: str = Field(description="The name of the Python class.")
python_module_name: str = Field(
description="The name of the Python module this class lives in."
)
full_name: str = Field(description="The full class namespace.")
# context_metadata: Optional[ContextMetadataModel] = Field(
# description="Context metadata for the class.", default=None
# )
_module_cache: ModuleType = PrivateAttr(default=None)
_cls_cache: Type = PrivateAttr(default=None)
def _retrieve_id(self) -> str:
return self.full_name
def _retrieve_data_to_hash(self) -> Any:
return self.full_name
def get_class(self) -> Type:
if self._cls_cache is None:
m = self.get_python_module()
self._cls_cache = getattr(m, self.python_class_name)
return self._cls_cache
def get_python_module(self) -> ModuleType:
if self._module_cache is None:
self._module_cache = importlib.import_module(self.python_module_name)
return self._module_cache
full_name: str
pydantic-field
required
¶The full class namespace.
python_class_name: str
pydantic-field
required
¶The name of the Python class.
python_module_name: str
pydantic-field
required
¶The name of the Python module this class lives in.
from_class(item_cls, attach_context_metadata=False)
classmethod
¶Source code in kiara/models/python_class.py
@classmethod
def from_class(cls, item_cls: Type, attach_context_metadata: bool = False):
cls_name = item_cls.__name__
module_name = item_cls.__module__
if module_name == "builtins":
full_name = cls_name
else:
full_name = f"{item_cls.__module__}.{item_cls.__name__}"
conf: Dict[str, Any] = {
"python_class_name": cls_name,
"python_module_name": module_name,
"full_name": full_name,
}
if attach_context_metadata:
raise NotImplementedError()
ctx_md = ContextMetadataModel.from_class(item_cls=item_cls)
conf["items"] = ctx_md
result = PythonClass.construct(**conf)
result._cls_cache = item_cls
return result
get_class(self)
¶Source code in kiara/models/python_class.py
def get_class(self) -> Type:
if self._cls_cache is None:
m = self.get_python_module()
self._cls_cache = getattr(m, self.python_class_name)
return self._cls_cache
get_python_module(self)
¶Source code in kiara/models/python_class.py
def get_python_module(self) -> ModuleType:
if self._module_cache is None:
self._module_cache = importlib.import_module(self.python_module_name)
return self._module_cache
rendering
special
¶
DataT
¶Classes¶
RenderScene (KiaraModel)
pydantic-model
¶Source code in kiara/models/rendering/__init__.py
class RenderScene(KiaraModel):
_kiara_model_id = "instance.render_scene"
title: str = Field(description="The title of this scene.")
disabled: bool = Field(
description="Whether this scene should be displayed as 'disabled' in a UI.",
default=False,
)
description: str = Field(
description="Description of what this scene renders.",
default=DEFAULT_NO_DESC_VALUE,
)
manifest_hash: str = Field(
description="The hash of the manifest of the referenced render scene."
)
render_config: Mapping[str, Any] = Field(
description="The inputs used with the referenced manifest.",
default_factory=dict,
)
related_scenes: Mapping[str, Union[None, "RenderScene"]] = Field(
description="Other render scenes, related to this one.", default_factory=dict
)
@validator("manifest_hash", pre=True)
def validate_manifest_hash(cls, value):
if hasattr(value, "manifest_hash"):
return value.manifest_hash # type: ignore
else:
return value
# @validator("description", pre=True)
# def validate_desc(cls, value):
# return DocumentationMetadataModel.create(value)
description: str
pydantic-field
¶Description of what this scene renders.
disabled: bool
pydantic-field
¶Whether this scene should be displayed as 'disabled' in a UI.
manifest_hash: str
pydantic-field
required
¶The hash of the manifest of the referenced render scene.
related_scenes: Mapping[str, Optional[RenderScene]]
pydantic-field
¶Other render scenes, related to this one.
render_config: Mapping[str, Any]
pydantic-field
¶The inputs used with the referenced manifest.
title: str
pydantic-field
required
¶The title of this scene.
validate_manifest_hash(value)
classmethod
¶Source code in kiara/models/rendering/__init__.py
@validator("manifest_hash", pre=True)
def validate_manifest_hash(cls, value):
if hasattr(value, "manifest_hash"):
return value.manifest_hash # type: ignore
else:
return value
RenderValueResult (KiaraModel)
pydantic-model
¶Object containing all the result properties of a 'render_value' operation.
Source code in kiara/models/rendering/__init__.py
class RenderValueResult(KiaraModel):
"""Object containing all the result properties of a 'render_value' operation."""
value_id: uuid.UUID = Field(description="The value that was rendered.")
render_config: Mapping[str, Any] = Field(
description="The config that was used to render this."
)
render_manifest: str = Field(
description="The id of the manifest that was used to render this."
)
related_scenes: Mapping[str, Union[None, RenderScene]] = Field(
description="Other render scenes, related to this one.", default_factory=dict
)
manifest_lookup: Dict[str, Manifest] = Field(
description="The manifests referenced in this model, indexed by the hashes.",
default_factory=dict,
)
rendered: Any = Field(description="The rendered object.")
def _retrieve_data_to_hash(self) -> EncodableType:
return {
"value_id": str(self.value_id),
"render_config": self.render_config,
"render_manifest": self.render_manifest,
}
manifest_lookup: Dict[str, kiara.models.module.manifest.Manifest]
pydantic-field
¶The manifests referenced in this model, indexed by the hashes.
related_scenes: Mapping[str, Optional[kiara.models.rendering.RenderScene]]
pydantic-field
¶Other render scenes, related to this one.
render_config: Mapping[str, Any]
pydantic-field
required
¶The config that was used to render this.
render_manifest: str
pydantic-field
required
¶The id of the manifest that was used to render this.
rendered: Any
pydantic-field
¶The rendered object.
value_id: UUID
pydantic-field
required
¶The value that was rendered.
values
¶
runtime_environment
special
¶
logger
¶
RuntimeEnvironment (KiaraModel)
pydantic-model
¶Source code in kiara/models/runtime_environment/__init__.py
class RuntimeEnvironment(KiaraModel):
class Config:
underscore_attrs_are_private = False
allow_mutation = False
@classmethod
def get_environment_type_name(cls) -> str:
env_type = cls.__fields__["environment_type"]
args = get_args(env_type.type_)
assert len(args) == 1
return args[0]
@classmethod
def create_environment_model(cls):
try:
type_name = cls.get_environment_type_name()
data = cls.retrieve_environment_data()
assert (
"environment_type" not in data.keys()
or data["environment_keys"] == type_name
)
data["environment_type"] = type_name
except Exception as e:
raise Exception(f"Can't create environment model for '{cls.__name__}': {e}")
return cls(**data)
def get_category_alias(self) -> str:
return f"{ENVIRONMENT_TYPE_CATEGORY_ID}.{self.environment_type}" # type: ignore
@classmethod
@abstractmethod
def retrieve_environment_data(cls) -> Dict[str, Any]:
pass
_env_hashes: Union[Mapping[str, str], None] = PrivateAttr(default=None)
def _create_renderable_for_field(
self, field_name: str, for_summary: bool = False
) -> Union[RenderableType, None]:
return extract_renderable(getattr(self, field_name))
def _retrieve_id(self) -> str:
return self.__class__.get_environment_type_name()
@property
def env_hashes(self) -> Mapping[str, str]:
if self._env_hashes is not None:
return self._env_hashes
result = {}
for k, v in self._retrieve_sub_profile_env_data().items():
_, cid = compute_cid(data=v)
result[k] = str(cid)
self._env_hashes = result
return self._env_hashes
def _retrieve_sub_profile_env_data(self) -> Mapping[str, Any]:
"""Return a dictionary with data that identifies one hash profile per key.
In most cases, this will only return a single-key dictionary containing all the data in that environment. But
in some cases one might want to have several sub-hash profiles, for example a list of Python packages with and
without version information, to have more fine-grained control about when to consider two environments functionally
equal.
"""
return {DEFAULT_ENV_HASH_KEY: self._retrieve_data_to_hash()}
def create_renderable(self, **config: Any) -> RenderableType:
summary = config.get("summary", False)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("field")
table.add_column("summary")
hashes_str = orjson_dumps(self.env_hashes, option=orjson.OPT_INDENT_2)
table.add_row(
"environment hashes", Syntax(hashes_str, "json", background_color="default")
)
for field_name, field in self.__fields__.items():
summary_item = self._create_renderable_for_field(
field_name, for_summary=summary
)
if summary_item is not None:
table.add_row(field_name, summary_item)
return table
env_hashes: Mapping[str, str]
property
readonly
¶
Config
¶create_environment_model()
classmethod
¶Source code in kiara/models/runtime_environment/__init__.py
@classmethod
def create_environment_model(cls):
try:
type_name = cls.get_environment_type_name()
data = cls.retrieve_environment_data()
assert (
"environment_type" not in data.keys()
or data["environment_keys"] == type_name
)
data["environment_type"] = type_name
except Exception as e:
raise Exception(f"Can't create environment model for '{cls.__name__}': {e}")
return cls(**data)
create_renderable(self, **config)
¶Source code in kiara/models/runtime_environment/__init__.py
def create_renderable(self, **config: Any) -> RenderableType:
summary = config.get("summary", False)
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("field")
table.add_column("summary")
hashes_str = orjson_dumps(self.env_hashes, option=orjson.OPT_INDENT_2)
table.add_row(
"environment hashes", Syntax(hashes_str, "json", background_color="default")
)
for field_name, field in self.__fields__.items():
summary_item = self._create_renderable_for_field(
field_name, for_summary=summary
)
if summary_item is not None:
table.add_row(field_name, summary_item)
return table
get_category_alias(self)
¶Source code in kiara/models/runtime_environment/__init__.py
def get_category_alias(self) -> str:
return f"{ENVIRONMENT_TYPE_CATEGORY_ID}.{self.environment_type}" # type: ignore
get_environment_type_name()
classmethod
¶Source code in kiara/models/runtime_environment/__init__.py
@classmethod
def get_environment_type_name(cls) -> str:
env_type = cls.__fields__["environment_type"]
args = get_args(env_type.type_)
assert len(args) == 1
return args[0]
retrieve_environment_data()
classmethod
¶Source code in kiara/models/runtime_environment/__init__.py
@classmethod
@abstractmethod
def retrieve_environment_data(cls) -> Dict[str, Any]:
pass
Modules¶
kiara
¶
KiaraTypesRuntimeEnvironment (RuntimeEnvironment)
pydantic-model
¶Source code in kiara/models/runtime_environment/kiara.py
class KiaraTypesRuntimeEnvironment(RuntimeEnvironment):
_kiara_model_id = "info.runtime.kiara_types"
environment_type: Literal["kiara_types"]
archive_types: ArchiveTypeClassesInfo = Field(
description="The available implemented store types."
)
metadata_types: MetadataTypeClassesInfo = Field(
description="The available metadata types."
)
@classmethod
def retrieve_environment_data(cls) -> Dict[str, Any]:
result: Dict[str, Any] = {}
result["metadata_types"] = find_metadata_models()
result["archive_types"] = find_archive_types()
return result
archive_types: ArchiveTypeClassesInfo
pydantic-field
required
¶The available implemented store types.
environment_type: Literal['kiara_types']
pydantic-field
required
¶metadata_types: MetadataTypeClassesInfo
pydantic-field
required
¶The available metadata types.
retrieve_environment_data()
classmethod
¶Source code in kiara/models/runtime_environment/kiara.py
@classmethod
def retrieve_environment_data(cls) -> Dict[str, Any]:
result: Dict[str, Any] = {}
result["metadata_types"] = find_metadata_models()
result["archive_types"] = find_archive_types()
return result
find_archive_types(alias=None, only_for_package=None)
¶Source code in kiara/models/runtime_environment/kiara.py
def find_archive_types(
alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> ArchiveTypeClassesInfo:
archive_types = find_all_archive_types()
kiara: Kiara = None # type: ignore
group: ArchiveTypeClassesInfo = ArchiveTypeClassesInfo.create_from_type_items( # type: ignore
kiara=kiara, group_title=alias, **archive_types
)
if only_for_package:
temp: Dict[str, TypeInfo] = {}
for key, info in group.item_infos.items():
if info.context.labels.get("package") == only_for_package:
temp[key] = info # type: ignore
group = ArchiveTypeClassesInfo.construct(
group_id=group.group_id, group_alias=group.group_alias, item_infos=temp # type: ignore
)
return group
operating_system
¶
OSRuntimeEnvironment (RuntimeEnvironment)
pydantic-model
¶Manages information about the OS this kiara instance is running in.
TODO: details for other OS's (mainly BSDs)¶
Source code in kiara/models/runtime_environment/operating_system.py
class OSRuntimeEnvironment(RuntimeEnvironment):
"""Manages information about the OS this kiara instance is running in.
# TODO: details for other OS's (mainly BSDs)
"""
_kiara_model_id = "info.runtime.os"
environment_type: typing.Literal["operating_system"]
operation_system: str = Field(description="The operation system name.")
platform: str = Field(description="The platform name.")
release: str = Field(description="The platform release name.")
version: str = Field(description="The platform version name.")
machine: str = Field(description="The architecture.")
os_specific: typing.Dict[str, typing.Any] = Field(
description="OS specific platform metadata.", default_factory=dict
)
@classmethod
def retrieve_environment_data(self) -> typing.Dict[str, typing.Any]:
os_specific: typing.Dict[str, typing.Any] = {}
platform_system = platform.system()
if platform_system == "Linux":
import distro
os_specific["distribution"] = {
"name": distro.name(),
"version": distro.version(),
"codename": distro.codename(),
}
elif platform_system == "Darwin":
mac_version = platform.mac_ver()
os_specific["mac_ver_release"] = mac_version[0]
os_specific["mac_ver_machine"] = mac_version[2]
result = {
"operation_system": os.name,
"platform": platform_system,
"release": platform.release(),
"version": platform.version(),
"machine": platform.machine(),
"os_specific": os_specific,
}
# if config.include_all_info:
# result["uname"] = platform.uname()._asdict()
return result
environment_type: Literal['operating_system']
pydantic-field
required
¶machine: str
pydantic-field
required
¶The architecture.
operation_system: str
pydantic-field
required
¶The operation system name.
os_specific: Dict[str, Any]
pydantic-field
¶OS specific platform metadata.
platform: str
pydantic-field
required
¶The platform name.
release: str
pydantic-field
required
¶The platform release name.
version: str
pydantic-field
required
¶The platform version name.
retrieve_environment_data()
classmethod
¶Source code in kiara/models/runtime_environment/operating_system.py
@classmethod
def retrieve_environment_data(self) -> typing.Dict[str, typing.Any]:
os_specific: typing.Dict[str, typing.Any] = {}
platform_system = platform.system()
if platform_system == "Linux":
import distro
os_specific["distribution"] = {
"name": distro.name(),
"version": distro.version(),
"codename": distro.codename(),
}
elif platform_system == "Darwin":
mac_version = platform.mac_ver()
os_specific["mac_ver_release"] = mac_version[0]
os_specific["mac_ver_machine"] = mac_version[2]
result = {
"operation_system": os.name,
"platform": platform_system,
"release": platform.release(),
"version": platform.version(),
"machine": platform.machine(),
"os_specific": os_specific,
}
# if config.include_all_info:
# result["uname"] = platform.uname()._asdict()
return result
python
¶
KiaraPluginsRuntimeEnvironment (RuntimeEnvironment)
pydantic-model
¶Source code in kiara/models/runtime_environment/python.py
class KiaraPluginsRuntimeEnvironment(RuntimeEnvironment):
_kiara_model_id = "info.runtime.kiara_plugins"
environment_type: Literal["kiara_plugins"]
kiara_plugins: List[PythonPackage] = Field(
description="The kiara plugin packages installed in the Python (virtual) environment."
)
def _create_renderable_for_field(
self, field_name: str, for_summary: bool = False
) -> Union[RenderableType, None]:
if field_name != "packages":
return extract_renderable(getattr(self, field_name))
if for_summary:
return ", ".join(p.name for p in self.kiara_plugins)
table = Table(show_header=True, box=box.SIMPLE)
table.add_column("package name")
table.add_column("version")
for package in self.kiara_plugins:
table.add_row(package.name, package.version)
return table
def _retrieve_sub_profile_env_data(self) -> Mapping[str, Any]:
only_packages = [p.name for p in self.kiara_plugins]
full = {k.name: k.version for k in self.kiara_plugins}
return {"package_names": only_packages, "packages": full}
@classmethod
def retrieve_environment_data(cls) -> Dict[str, Any]:
packages = []
all_packages = find_all_distributions()
for name, pkgs in all_packages.items():
for pkg in pkgs:
if not pkg.startswith("kiara_plugin.") and not pkg.startswith(
"kiara-plugin."
):
continue
dist = distribution(pkg)
packages.append({"name": pkg, "version": dist.version})
result: Dict[str, Any] = {
"kiara_plugins": sorted(packages, key=lambda x: x["name"]),
}
# if config.include_all_info:
# import sysconfig
# result["python_config"] = sysconfig.get_config_vars()
return result
environment_type: Literal['kiara_plugins']
pydantic-field
required
¶kiara_plugins: List[kiara.models.runtime_environment.python.PythonPackage]
pydantic-field
required
¶The kiara plugin packages installed in the Python (virtual) environment.
retrieve_environment_data()
classmethod
¶Source code in kiara/models/runtime_environment/python.py
@classmethod
def retrieve_environment_data(cls) -> Dict[str, Any]:
packages = []
all_packages = find_all_distributions()
for name, pkgs in all_packages.items():
for pkg in pkgs:
if not pkg.startswith("kiara_plugin.") and not pkg.startswith(
"kiara-plugin."
):
continue
dist = distribution(pkg)
packages.append({"name": pkg, "version": dist.version})
result: Dict[str, Any] = {
"kiara_plugins": sorted(packages, key=lambda x: x["name"]),
}
# if config.include_all_info:
# import sysconfig
# result["python_config"] = sysconfig.get_config_vars()
return result
PythonPackage (BaseModel)
pydantic-model
¶Source code in kiara/models/runtime_environment/python.py
class PythonPackage(BaseModel):
name: str = Field(description="The name of the Python package.")
version: str = Field(description="The version of the package.")
PythonRuntimeEnvironment (RuntimeEnvironment)
pydantic-model
¶Source code in kiara/models/runtime_environment/python.py
class PythonRuntimeEnvironment(RuntimeEnvironment):
_kiara_model_id = "info.runtime.python"
environment_type: Literal["python"]
python_version: str = Field(description="The version of Python.")
packages: List[PythonPackage] = Field(
description="The packages installed in the Python (virtual) environment."
)
# python_config: typing.Dict[str, str] = Field(
# description="Configuration details about the Python installation."
# )
def _create_renderable_for_field(
self, field_name: str, for_summary: bool = False
) -> Union[RenderableType, None]:
if field_name != "packages":
return extract_renderable(getattr(self, field_name))
if for_summary:
return ", ".join(p.name for p in self.packages)
table = Table(show_header=True, box=box.SIMPLE)
table.add_column("package name")
table.add_column("version")
for package in self.packages:
table.add_row(package.name, package.version)
return table
def _retrieve_sub_profile_env_data(self) -> Mapping[str, Any]:
only_packages = [p.name for p in self.packages]
full = {k.name: k.version for k in self.packages}
return {
"package_names": only_packages,
"packages": full,
"package_names_incl_python_version": {
"python_version": self.python_version,
"packages": only_packages,
},
"packages_incl_python_version": {
"python_version": self.python_version,
"packages": full,
},
}
@classmethod
def retrieve_environment_data(cls) -> Dict[str, Any]:
packages = []
all_packages = find_all_distributions()
for name, pkgs in all_packages.items():
for pkg in pkgs:
dist = distribution(pkg)
packages.append({"name": pkg, "version": dist.version})
result: Dict[str, Any] = {
"python_version": sys.version,
"packages": sorted(packages, key=lambda x: x["name"]),
}
# if config.include_all_info:
# import sysconfig
# result["python_config"] = sysconfig.get_config_vars()
return result
environment_type: Literal['python']
pydantic-field
required
¶packages: List[kiara.models.runtime_environment.python.PythonPackage]
pydantic-field
required
¶The packages installed in the Python (virtual) environment.
python_version: str
pydantic-field
required
¶The version of Python.
retrieve_environment_data()
classmethod
¶Source code in kiara/models/runtime_environment/python.py
@classmethod
def retrieve_environment_data(cls) -> Dict[str, Any]:
packages = []
all_packages = find_all_distributions()
for name, pkgs in all_packages.items():
for pkg in pkgs:
dist = distribution(pkg)
packages.append({"name": pkg, "version": dist.version})
result: Dict[str, Any] = {
"python_version": sys.version,
"packages": sorted(packages, key=lambda x: x["name"]),
}
# if config.include_all_info:
# import sysconfig
# result["python_config"] = sysconfig.get_config_vars()
return result
find_all_distributions()
¶Source code in kiara/models/runtime_environment/python.py
@lru_cache()
def find_all_distributions():
all_packages = packages_distributions()
return all_packages
values
special
¶
DEFAULT_SCALAR_DATATYPE_CHARACTERISTICS
¶Classes¶
DataTypeCharacteristics (BaseModel)
pydantic-model
¶Source code in kiara/models/values/__init__.py
class DataTypeCharacteristics(BaseModel):
is_scalar: bool = Field(
description="Whether the data desribed by this data type behaves like a skalar.",
default=False,
)
is_json_serializable: bool = Field(
description="Whether the data can be serialized to json without information loss.",
default=False,
)
ValueStatus (Enum)
¶Modules¶
data_type
¶lineage
¶COLOR_LIST
¶
ValueLineage (JupyterMixin)
¶Source code in kiara/models/values/lineage.py
class ValueLineage(JupyterMixin):
@classmethod
def from_value(cls, value: Value) -> "ValueLineage":
pass
def __init__(self, kiara: "Kiara", value: Value):
self._value: Value = value
self._kiara: Kiara = kiara
self._full_graph: Union[None, DiGraph] = None
self._module_graph: Union[None, DiGraph] = None
@property
def full_graph(self) -> DiGraph:
if self._full_graph is not None:
return self._full_graph
self._full_graph = create_lineage_graph(kiara=self._kiara, value=self._value)
return self._full_graph
@property
def module_graph(self) -> DiGraph:
if self._module_graph is not None:
return self._module_graph
self._module_graph = create_lineage_graph_modules(
kiara=self._kiara, value=self._value
).reverse()
return self._module_graph
def create_renderable(self, **config: Any) -> RenderableType:
include_ids: bool = config.get("include_ids", False)
tree = fill_renderable_lineage_tree(
kiara=self._kiara, pedigree=self._value.pedigree, include_ids=include_ids
)
return tree
def __rich_console__(
self, console: Console, options: ConsoleOptions
) -> RenderResult:
yield self.create_renderable()
full_graph: DiGraph
property
readonly
¶module_graph: DiGraph
property
readonly
¶create_renderable(self, **config)
¶Source code in kiara/models/values/lineage.py
def create_renderable(self, **config: Any) -> RenderableType:
include_ids: bool = config.get("include_ids", False)
tree = fill_renderable_lineage_tree(
kiara=self._kiara, pedigree=self._value.pedigree, include_ids=include_ids
)
return tree
from_value(value)
classmethod
¶Source code in kiara/models/values/lineage.py
@classmethod
def from_value(cls, value: Value) -> "ValueLineage":
pass
create_lineage_graph(kiara, value, graph=None, parent=None, level=1)
¶Source code in kiara/models/values/lineage.py
def create_lineage_graph(
kiara: "Kiara",
value: Value,
graph: Union[DiGraph, None] = None,
parent: Union[None, str] = None,
level: int = 1,
) -> DiGraph:
if graph is None:
graph = DiGraph()
graph.add_node(
f"value:{value.value_id}",
data_type=value.data_type_name,
label="root_value",
node_type="value",
data_type_config=value.data_type_config,
level=1,
)
parent = f"value:{value.value_id}"
module_id = f"module:{value.pedigree.job_hash}"
module_label = f"module:{value.pedigree.module_type}"
graph.add_node(
module_id,
module_type=value.pedigree.module_type,
module_config=value.pedigree.module_config,
label=module_label,
node_type="operation",
level=(level * 2) + 1,
)
graph.add_edge(
parent,
module_id,
id=f"{parent}:{module_id}",
field_name=value.pedigree_output_name,
label=value.pedigree_output_name,
)
for input_name in sorted(value.pedigree.inputs.keys()):
child_value_id = value.pedigree.inputs[input_name]
child_value = kiara.data_registry.get_value(child_value_id)
input_id = f"value:{child_value.value_id}"
input_label = f"{input_name}:{input_name}"
graph.add_node(
input_id,
label=input_label,
node_type="value",
data_type=child_value.data_type_name,
data_type_config=child_value.data_type_config,
level=(level * 2) + 2,
)
graph.add_edge(
module_id,
input_id,
id=f"{module_id}:{input_id}",
field_name=input_name,
label=input_name,
)
if child_value.pedigree != ORPHAN:
create_lineage_graph(
kiara=kiara,
value=child_value,
graph=graph,
parent=input_id,
level=level + 1,
)
return graph
create_lineage_graph_modules(kiara, value, graph=None, parent=None, input_field=None, level=1)
¶Source code in kiara/models/values/lineage.py
def create_lineage_graph_modules(
kiara: "Kiara",
value: Value,
graph: Union[DiGraph, None] = None,
parent: Union[None, str] = None,
input_field: Union[None, str] = None,
level: int = 1,
) -> DiGraph:
if graph is None:
graph = DiGraph()
graph.add_node(
f"value:{value.value_id}",
data_type=value.data_type_name,
label="[this value]",
node_type="value",
data_type_config=value.data_type_config,
level=1,
)
module_id = f"module:{value.pedigree.job_hash}"
module_label = value.pedigree.module_type
graph.add_node(
module_id,
module_type=value.pedigree.module_type,
module_config=value.pedigree.module_config,
label=module_label,
node_type="operation",
level=(level * 2) + 1,
)
if parent is None:
parent = f"value:{value.value_id}"
graph.add_edge(
parent,
module_id,
id=f"{parent}:{module_id}",
field_name=value.pedigree_output_name,
label=f"{value.pedigree_output_name} ({value.data_type_name})",
)
else:
assert input_field is not None
graph.add_edge(
parent,
module_id,
id=f"{parent}:{input_field}",
field_name=input_field,
label=f"{input_field} ({value.data_type_name})",
)
for input_name in sorted(value.pedigree.inputs.keys()):
child_value_id = value.pedigree.inputs[input_name]
child_value = kiara.data_registry.get_value(child_value_id)
if child_value.pedigree != ORPHAN:
create_lineage_graph_modules(
kiara=kiara,
value=child_value,
graph=graph,
parent=module_id,
input_field=input_name,
level=level + 1,
)
else:
input_id = f"value:{child_value.value_id}"
input_label = f"{input_name} ({child_value.data_type_name})"
graph.add_node(
input_id,
label=input_label,
node_type="value",
data_type=child_value.data_type_name,
data_type_config=child_value.data_type_config,
level=(level * 2) + 2,
)
graph.add_edge(
module_id,
input_id,
id=f"{module_id}:{input_id}",
field_name=input_name,
label=f"{input_name} ({child_value.data_type_name})",
)
return graph
fill_renderable_lineage_tree(kiara, pedigree, node=None, include_ids=False, level=0)
¶Source code in kiara/models/values/lineage.py
def fill_renderable_lineage_tree(
kiara: "Kiara",
pedigree: ValuePedigree,
node: Union[Tree, None] = None,
include_ids: bool = False,
level: int = 0,
):
color = COLOR_LIST[level % len(COLOR_LIST)]
title = f"[b {color}]{pedigree.module_type}[/b {color}]"
if node is None:
main = Tree(title)
else:
main = node.add(title)
for input_name in sorted(pedigree.inputs.keys()):
child_value_id = pedigree.inputs[input_name]
child_value = kiara.data_registry.get_value(child_value_id)
value_type = child_value.data_type_name
if include_ids:
v_id_str = f" = {child_value.value_id}"
else:
v_id_str = ""
input_node = main.add(
f"input: [i {color}]{input_name} ({value_type})[/i {color}]{v_id_str}"
)
if child_value.pedigree != ORPHAN:
fill_renderable_lineage_tree(
kiara=kiara,
pedigree=child_value.pedigree,
node=input_node,
level=level + 1,
include_ids=include_ids,
)
return main
matchers
¶
ValueMatcher (KiaraModel)
pydantic-model
¶An object describing requirements values should satisfy in order to be included in a query result.
Source code in kiara/models/values/matchers.py
class ValueMatcher(KiaraModel):
"""An object describing requirements values should satisfy in order to be included in a query result."""
@classmethod
def create_matcher(self, **match_options: Any):
m = ValueMatcher(**match_options)
return m
data_types: List[str] = Field(description="The data type.", default_factory=list)
allow_sub_types: bool = Field(description="Allow subtypes.", default=True)
min_size: int = Field(description="The minimum size for the dataset.", default=0)
max_size: Union[None, int] = Field(
description="The maximum size for the dataset.", default=None
)
allow_internal: bool = Field(
description="Allow internal data types.", default=False
)
has_alias: bool = Field(
description="Value must have at least one alias.", default=True
)
def is_match(self, value: Value, kiara: "Kiara") -> bool:
if self.data_types:
match = False
if not self.allow_sub_types:
for data_type in self.data_types:
if data_type == value.data_type_name:
match = True
break
else:
if value.data_type_name not in kiara.type_registry.data_type_names:
return False
lineage = kiara.type_registry.get_type_lineage(value.data_type_name)
for data_type in self.data_types:
if data_type in lineage:
match = True
break
if not match:
return False
if self.min_size:
if value.value_size < self.min_size:
return False
if self.max_size:
if value.value_size > self.max_size:
return False
if not self.allow_internal:
if kiara.type_registry.is_internal_type(
data_type_name=value.data_type_name
):
return False
if self.has_alias:
aliases = kiara.alias_registry.find_aliases_for_value_id(
value_id=value.value_id
)
if not aliases:
return False
return True
allow_internal: bool
pydantic-field
¶Allow internal data types.
allow_sub_types: bool
pydantic-field
¶Allow subtypes.
data_types: List[str]
pydantic-field
¶The data type.
has_alias: bool
pydantic-field
¶Value must have at least one alias.
max_size: int
pydantic-field
¶The maximum size for the dataset.
min_size: int
pydantic-field
¶The minimum size for the dataset.
create_matcher(**match_options)
classmethod
¶Source code in kiara/models/values/matchers.py
@classmethod
def create_matcher(self, **match_options: Any):
m = ValueMatcher(**match_options)
return m
is_match(self, value, kiara)
¶Source code in kiara/models/values/matchers.py
def is_match(self, value: Value, kiara: "Kiara") -> bool:
if self.data_types:
match = False
if not self.allow_sub_types:
for data_type in self.data_types:
if data_type == value.data_type_name:
match = True
break
else:
if value.data_type_name not in kiara.type_registry.data_type_names:
return False
lineage = kiara.type_registry.get_type_lineage(value.data_type_name)
for data_type in self.data_types:
if data_type in lineage:
match = True
break
if not match:
return False
if self.min_size:
if value.value_size < self.min_size:
return False
if self.max_size:
if value.value_size > self.max_size:
return False
if not self.allow_internal:
if kiara.type_registry.is_internal_type(
data_type_name=value.data_type_name
):
return False
if self.has_alias:
aliases = kiara.alias_registry.find_aliases_for_value_id(
value_id=value.value_id
)
if not aliases:
return False
return True
value
¶ORPHAN
¶SERIALIZE_TYPES
¶log
¶yaml
¶
DataTypeInfo (KiaraModel)
pydantic-model
¶Source code in kiara/models/values/value.py
class DataTypeInfo(KiaraModel):
_kiara_model_id = "info.data_type_instance"
data_type_name: str = Field(description="The registered name of this data type.")
data_type_config: Mapping[str, Any] = Field(
description="The (optional) configuration for this data type.",
default_factory=dict,
)
characteristics: DataTypeCharacteristics = Field(
description="Characteristics of this data type."
)
data_type_class: PythonClass = Field(
description="The python class that is associated with this model."
)
_data_type_instance: "DataType" = PrivateAttr(default=None)
@property
def data_type_instance(self) -> "DataType":
if self._data_type_instance is not None:
return self._data_type_instance
self._data_type_instance = self.data_type_class.get_class()(
**self.data_type_config
)
return self._data_type_instance
characteristics: DataTypeCharacteristics
pydantic-field
required
¶Characteristics of this data type.
data_type_class: PythonClass
pydantic-field
required
¶The python class that is associated with this model.
data_type_config: Mapping[str, Any]
pydantic-field
¶The (optional) configuration for this data type.
data_type_instance: DataType
property
readonly
¶data_type_name: str
pydantic-field
required
¶The registered name of this data type.
PersistedData (SerializedData)
pydantic-model
¶Source code in kiara/models/values/value.py
class PersistedData(SerializedData):
_kiara_model_id = "instance.persisted_data"
archive_id: uuid.UUID = Field(
description="The id of the store that persisted the data."
)
chunk_id_map: Mapping[str, SerializedChunkIDs] = Field(
description="Reference-ids that resolve to the values' serialized chunks."
)
def get_keys(self) -> Iterable[str]:
return self.chunk_id_map.keys()
def get_serialized_data(self, key: str) -> SerializedChunks:
return self.chunk_id_map[key]
archive_id: UUID
pydantic-field
required
¶The id of the store that persisted the data.
chunk_id_map: Mapping[str, kiara.models.values.value.SerializedChunkIDs]
pydantic-field
required
¶Reference-ids that resolve to the values' serialized chunks.
get_keys(self)
¶Source code in kiara/models/values/value.py
def get_keys(self) -> Iterable[str]:
return self.chunk_id_map.keys()
get_serialized_data(self, key)
¶Source code in kiara/models/values/value.py
def get_serialized_data(self, key: str) -> SerializedChunks:
return self.chunk_id_map[key]
SerializationMetadata (KiaraModel)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializationMetadata(KiaraModel):
_kiara_model_id = "metadata.serialized_data"
environment: Mapping[str, int] = Field(
description="Hash(es) for the environments the value was created/serialized.",
default_factory=dict,
)
deserialize: Mapping[str, Manifest] = Field(
description="Suggested manifest configs to use to de-serialize the data.",
default_factory=dict,
)
SerializationResult (SerializedData)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializationResult(SerializedData):
_kiara_model_id = "instance.serialization_result"
data: Dict[
str,
Union[
SerializedBytes,
SerializedListOfBytes,
SerializedFile,
SerializedFiles,
SerializedInlineJson,
],
] = Field(
description="One or several byte arrays representing the serialized state of the value."
)
def get_keys(self) -> Iterable[str]:
return self.data.keys()
def get_serialized_data(self, key: str) -> SerializedChunks:
return self.data[key]
@root_validator(pre=True)
def validate_data(cls, values):
codec = values.get("codec", None)
if codec is None:
codec = "sha2-256"
values["hash_codec"] = codec
v = values.get("data")
assert isinstance(v, Mapping)
result = {}
for field_name, data in v.items():
if isinstance(data, SerializedChunks):
result[field_name] = data
elif isinstance(data, Mapping):
s_type = data.get("type", None)
if not s_type:
raise ValueError(
f"Invalid serialized data config, missing 'type' key: {data}"
)
if s_type not in SERIALIZE_TYPES.keys():
raise ValueError(
f"Invalid serialized data type '{s_type}'. Allowed types: {', '.join(SERIALIZE_TYPES.keys())}"
)
assert s_type != "chunk-ids"
cls = SERIALIZE_TYPES[s_type]
result[field_name] = cls(**data)
values["data"] = result
return values
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key")
table.add_column("value")
table.add_row("data_type", self.data_type)
_config = Syntax(
orjson_dumps(self.data_type_config), "json", background_color="default"
)
table.add_row("data_type_config", _config)
data_fields = {}
for field, model in self.data.items():
data_fields[field] = {"type": model.type}
data_json = Syntax(
orjson_dumps(data_fields), "json", background_color="default"
)
table.add_row("data", data_json)
table.add_row("size", str(self.data_size))
table.add_row("hash", self.instance_id)
return table
def __repr__(self):
return f"{self.__class__.__name__}(type={self.data_type} size={self.data_size})"
def __str__(self):
return self.__repr__()
data: Dict[str, Union[kiara.models.values.value.SerializedBytes, kiara.models.values.value.SerializedListOfBytes, kiara.models.values.value.SerializedFile, kiara.models.values.value.SerializedFiles, kiara.models.values.value.SerializedInlineJson]]
pydantic-field
required
¶One or several byte arrays representing the serialized state of the value.
create_renderable(self, **config)
¶Source code in kiara/models/values/value.py
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key")
table.add_column("value")
table.add_row("data_type", self.data_type)
_config = Syntax(
orjson_dumps(self.data_type_config), "json", background_color="default"
)
table.add_row("data_type_config", _config)
data_fields = {}
for field, model in self.data.items():
data_fields[field] = {"type": model.type}
data_json = Syntax(
orjson_dumps(data_fields), "json", background_color="default"
)
table.add_row("data", data_json)
table.add_row("size", str(self.data_size))
table.add_row("hash", self.instance_id)
return table
get_keys(self)
¶Source code in kiara/models/values/value.py
def get_keys(self) -> Iterable[str]:
return self.data.keys()
get_serialized_data(self, key)
¶Source code in kiara/models/values/value.py
def get_serialized_data(self, key: str) -> SerializedChunks:
return self.data[key]
validate_data(values)
classmethod
¶Source code in kiara/models/values/value.py
@root_validator(pre=True)
def validate_data(cls, values):
codec = values.get("codec", None)
if codec is None:
codec = "sha2-256"
values["hash_codec"] = codec
v = values.get("data")
assert isinstance(v, Mapping)
result = {}
for field_name, data in v.items():
if isinstance(data, SerializedChunks):
result[field_name] = data
elif isinstance(data, Mapping):
s_type = data.get("type", None)
if not s_type:
raise ValueError(
f"Invalid serialized data config, missing 'type' key: {data}"
)
if s_type not in SERIALIZE_TYPES.keys():
raise ValueError(
f"Invalid serialized data type '{s_type}'. Allowed types: {', '.join(SERIALIZE_TYPES.keys())}"
)
assert s_type != "chunk-ids"
cls = SERIALIZE_TYPES[s_type]
result[field_name] = cls(**data)
values["data"] = result
return values
SerializedBytes (SerializedPreStoreChunks)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializedBytes(SerializedPreStoreChunks):
type: Literal["chunk"] = "chunk"
chunk: bytes = Field(description="A byte-array")
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if as_files is False:
return [self.chunk]
else:
if as_files is True:
file = None
elif isinstance(as_files, str):
file = as_files
else:
assert len(as_files) == 1
file = as_files[0]
path = self._store_bytes_to_file([self.chunk], file=file)
return path
def get_number_of_chunks(self) -> int:
return 1
def _get_size(self) -> int:
return len(self.chunk)
def _create_cids(self, hash_codec: str) -> Sequence[CID]:
return [self._create_cid_from_chunk(self.chunk, hash_codec=hash_codec)]
chunk: bytes
pydantic-field
required
¶A byte-array
type: Literal['chunk']
pydantic-field
¶get_chunks(self, as_files=True, symlink_ok=True)
¶Retrieve the chunks belonging to this data instance.
If 'as_file' is False, return the data as bytes. If set to 'True' store it to an arbitrary location (or use an existing one), and return the path to that file. If 'as_file' is a string, write the data (bytes) into a new file using the string as path. If 'symlink_ok' is set to True, symlinking an existing file to the value of 'as_file' is also ok, otherwise copy the content.
Source code in kiara/models/values/value.py
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if as_files is False:
return [self.chunk]
else:
if as_files is True:
file = None
elif isinstance(as_files, str):
file = as_files
else:
assert len(as_files) == 1
file = as_files[0]
path = self._store_bytes_to_file([self.chunk], file=file)
return path
get_number_of_chunks(self)
¶Source code in kiara/models/values/value.py
def get_number_of_chunks(self) -> int:
return 1
SerializedChunkIDs (SerializedChunks)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializedChunkIDs(SerializedChunks):
type: Literal["chunk-ids"] = "chunk-ids"
chunk_id_list: List[str] = Field(
description="A list of chunk ids, which will be resolved via the attached data registry."
)
archive_id: Union[uuid.UUID, None] = Field(
description="The preferred data archive to get the chunks from."
)
size: int = Field(description="The size of all chunks combined.")
_data_registry: "DataRegistry" = PrivateAttr(default=None)
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if isinstance(as_files, (bool, str)):
return (
self._data_registry.retrieve_chunk(
chunk_id=chunk,
archive_id=self.archive_id,
as_file=as_files,
symlink_ok=symlink_ok,
)
for chunk in self.chunk_id_list
)
else:
result = []
for idx, chunk_id in enumerate(self.chunk_id_list):
file = as_files[idx]
self._data_registry.retrieve_chunk(
chunk_id=chunk_id,
archive_id=self.archive_id,
as_file=file,
symlink_ok=symlink_ok,
)
result.append(file)
return result
def get_number_of_chunks(self) -> int:
return len(self.chunk_id_list)
def _get_size(self) -> int:
return self.size
def _create_cids(self, hash_codec: str) -> Sequence[CID]:
result = []
for chunk_id in self.chunk_id_list:
cid = CID.decode(chunk_id)
result.append(cid)
return result
archive_id: UUID
pydantic-field
¶The preferred data archive to get the chunks from.
chunk_id_list: List[str]
pydantic-field
required
¶A list of chunk ids, which will be resolved via the attached data registry.
size: int
pydantic-field
required
¶The size of all chunks combined.
type: Literal['chunk-ids']
pydantic-field
¶get_chunks(self, as_files=True, symlink_ok=True)
¶Retrieve the chunks belonging to this data instance.
If 'as_file' is False, return the data as bytes. If set to 'True' store it to an arbitrary location (or use an existing one), and return the path to that file. If 'as_file' is a string, write the data (bytes) into a new file using the string as path. If 'symlink_ok' is set to True, symlinking an existing file to the value of 'as_file' is also ok, otherwise copy the content.
Source code in kiara/models/values/value.py
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if isinstance(as_files, (bool, str)):
return (
self._data_registry.retrieve_chunk(
chunk_id=chunk,
archive_id=self.archive_id,
as_file=as_files,
symlink_ok=symlink_ok,
)
for chunk in self.chunk_id_list
)
else:
result = []
for idx, chunk_id in enumerate(self.chunk_id_list):
file = as_files[idx]
self._data_registry.retrieve_chunk(
chunk_id=chunk_id,
archive_id=self.archive_id,
as_file=file,
symlink_ok=symlink_ok,
)
result.append(file)
return result
get_number_of_chunks(self)
¶Source code in kiara/models/values/value.py
def get_number_of_chunks(self) -> int:
return len(self.chunk_id_list)
SerializedChunks (BaseModel, ABC)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializedChunks(BaseModel, abc.ABC):
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
extra = Extra.forbid
_size_cache: Union[int, None] = PrivateAttr(default=None)
_hashes_cache: Dict[str, Sequence[CID]] = PrivateAttr(default_factory=dict)
@abc.abstractmethod
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
"""Retrieve the chunks belonging to this data instance.
If 'as_file' is False, return the data as bytes. If set to 'True' store it to an arbitrary location (or use
an existing one), and return the path to that file. If 'as_file' is a string, write the data (bytes) into
a new file using the string as path. If 'symlink_ok' is set to True, symlinking an existing file to the value of
'as_file' is also ok, otherwise copy the content.
"""
@abc.abstractmethod
def get_number_of_chunks(self) -> int:
pass
@abc.abstractmethod
def _get_size(self) -> int:
pass
@abc.abstractmethod
def _create_cids(self, hash_codec: str) -> Sequence[CID]:
pass
def get_size(self) -> int:
if self._size_cache is None:
self._size_cache = self._get_size()
return self._size_cache
def get_cids(self, hash_codec: str) -> Sequence[CID]:
if self._hashes_cache.get(hash_codec, None) is None:
self._hashes_cache[hash_codec] = self._create_cids(hash_codec=hash_codec)
return self._hashes_cache[hash_codec]
def _store_bytes_to_file(
self, chunks: Iterable[bytes], file: Union[str, None] = None
) -> str:
"Utility method to store bytes to a file."
if file is None:
file_desc, file = tempfile.mkstemp()
def del_temp_file():
os.remove(file)
atexit.register(del_temp_file)
else:
if os.path.exists(file):
raise Exception(f"Can't write to file, file exists: {file}")
file_desc = os.open(file, 0o600)
with os.fdopen(file_desc, "wb") as tmp:
for chunk in chunks:
tmp.write(chunk)
return file
def _read_bytes_from_file(self, file: str) -> bytes:
with open(file, "rb") as f:
content = f.read()
return content
Config
¶Source code in kiara/models/values/value.py
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
extra = Extra.forbid
extra
¶json_loads
¶json_dumps(v, *, default=None, **args)
¶Source code in kiara/models/values/value.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
get_chunks(self, as_files=True, symlink_ok=True)
¶Retrieve the chunks belonging to this data instance.
If 'as_file' is False, return the data as bytes. If set to 'True' store it to an arbitrary location (or use an existing one), and return the path to that file. If 'as_file' is a string, write the data (bytes) into a new file using the string as path. If 'symlink_ok' is set to True, symlinking an existing file to the value of 'as_file' is also ok, otherwise copy the content.
Source code in kiara/models/values/value.py
@abc.abstractmethod
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
"""Retrieve the chunks belonging to this data instance.
If 'as_file' is False, return the data as bytes. If set to 'True' store it to an arbitrary location (or use
an existing one), and return the path to that file. If 'as_file' is a string, write the data (bytes) into
a new file using the string as path. If 'symlink_ok' is set to True, symlinking an existing file to the value of
'as_file' is also ok, otherwise copy the content.
"""
get_cids(self, hash_codec)
¶Source code in kiara/models/values/value.py
def get_cids(self, hash_codec: str) -> Sequence[CID]:
if self._hashes_cache.get(hash_codec, None) is None:
self._hashes_cache[hash_codec] = self._create_cids(hash_codec=hash_codec)
return self._hashes_cache[hash_codec]
get_number_of_chunks(self)
¶Source code in kiara/models/values/value.py
@abc.abstractmethod
def get_number_of_chunks(self) -> int:
pass
get_size(self)
¶Source code in kiara/models/values/value.py
def get_size(self) -> int:
if self._size_cache is None:
self._size_cache = self._get_size()
return self._size_cache
SerializedData (KiaraModel)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializedData(KiaraModel):
data_type: str = Field(
description="The name of the data type for this serialized value."
)
data_type_config: Mapping[str, Any] = Field(
description="The (optional) config for the data type for this serialized value.",
default_factory=dict,
)
serialization_profile: str = Field(
description="An identifying name for the serialization method used."
)
metadata: SerializationMetadata = Field(
description="Optional metadata describing aspects of the serialization used.",
default_factory=dict,
)
hash_codec: str = Field(
description="The codec used to hash the value.", default="sha2-256"
)
_cids_cache: Dict[str, Sequence[CID]] = PrivateAttr(default_factory=dict)
_cached_data_size: Union[int, None] = PrivateAttr(default=None)
_cached_dag: Union[Dict[str, Sequence[CID]], None] = PrivateAttr(default=None)
# _cached_cid: Optional[CID] = PrivateAttr(default=None)
def _retrieve_data_to_hash(self) -> Any:
return self.dag
@property
def data_size(self) -> int:
if self._cached_data_size is not None:
return self._cached_data_size
size = 0
for k in self.get_keys():
model = self.get_serialized_data(k)
size = size + model.get_size()
self._cached_data_size = size
return self._cached_data_size
@abc.abstractmethod
def get_keys(self) -> Iterable[str]:
pass
@abc.abstractmethod
def get_serialized_data(self, key: str) -> SerializedChunks:
pass
def get_cids_for_key(self, key) -> Sequence[CID]:
if key in self._cids_cache.keys():
return self._cids_cache[key]
model = self.get_serialized_data(key)
self._cids_cache[key] = model.get_cids(hash_codec=self.hash_codec)
return self._cids_cache[key]
@property
def dag(self) -> Mapping[str, Sequence[CID]]:
if self._cached_dag is not None:
return self._cached_dag
dag: Dict[str, Sequence[CID]] = {}
for key in self.get_keys():
dag[key] = self.get_cids_for_key(key)
self._cached_dag = dag
return self._cached_dag
dag: Mapping[str, Sequence[multiformats.cid.CID]]
property
readonly
¶data_size: int
property
readonly
¶data_type: str
pydantic-field
required
¶The name of the data type for this serialized value.
data_type_config: Mapping[str, Any]
pydantic-field
¶The (optional) config for the data type for this serialized value.
hash_codec: str
pydantic-field
¶The codec used to hash the value.
metadata: SerializationMetadata
pydantic-field
¶Optional metadata describing aspects of the serialization used.
serialization_profile: str
pydantic-field
required
¶An identifying name for the serialization method used.
get_cids_for_key(self, key)
¶Source code in kiara/models/values/value.py
def get_cids_for_key(self, key) -> Sequence[CID]:
if key in self._cids_cache.keys():
return self._cids_cache[key]
model = self.get_serialized_data(key)
self._cids_cache[key] = model.get_cids(hash_codec=self.hash_codec)
return self._cids_cache[key]
get_keys(self)
¶Source code in kiara/models/values/value.py
@abc.abstractmethod
def get_keys(self) -> Iterable[str]:
pass
get_serialized_data(self, key)
¶Source code in kiara/models/values/value.py
@abc.abstractmethod
def get_serialized_data(self, key: str) -> SerializedChunks:
pass
SerializedFile (SerializedPreStoreChunks)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializedFile(SerializedPreStoreChunks):
type: Literal["file"] = "file"
file: str = Field(description="A path to a file containing the serialized data.")
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if as_files is False:
chunk = self._read_bytes_from_file(self.file)
return [chunk]
else:
if as_files is True:
return [self.file]
else:
if isinstance(as_files, str):
file = as_files
else:
assert len(as_files) == 1
file = as_files[0]
if os.path.exists(file):
raise Exception(f"Can't write to file '{file}': file exists.")
if symlink_ok:
os.symlink(self.file, file)
return [file]
else:
raise NotImplementedError()
def get_number_of_chunks(self) -> int:
return 1
def _get_size(self) -> int:
return os.path.getsize(os.path.realpath(self.file))
def _create_cids(self, hash_codec: str) -> Sequence[CID]:
return [self._create_cid_from_file(self.file, hash_codec=hash_codec)]
file: str
pydantic-field
required
¶A path to a file containing the serialized data.
type: Literal['file']
pydantic-field
¶get_chunks(self, as_files=True, symlink_ok=True)
¶Retrieve the chunks belonging to this data instance.
If 'as_file' is False, return the data as bytes. If set to 'True' store it to an arbitrary location (or use an existing one), and return the path to that file. If 'as_file' is a string, write the data (bytes) into a new file using the string as path. If 'symlink_ok' is set to True, symlinking an existing file to the value of 'as_file' is also ok, otherwise copy the content.
Source code in kiara/models/values/value.py
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if as_files is False:
chunk = self._read_bytes_from_file(self.file)
return [chunk]
else:
if as_files is True:
return [self.file]
else:
if isinstance(as_files, str):
file = as_files
else:
assert len(as_files) == 1
file = as_files[0]
if os.path.exists(file):
raise Exception(f"Can't write to file '{file}': file exists.")
if symlink_ok:
os.symlink(self.file, file)
return [file]
else:
raise NotImplementedError()
get_number_of_chunks(self)
¶Source code in kiara/models/values/value.py
def get_number_of_chunks(self) -> int:
return 1
SerializedFiles (SerializedPreStoreChunks)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializedFiles(SerializedPreStoreChunks):
type: Literal["files"] = "files"
files: List[str] = Field(
description="A list of strings, pointing to files containing parts of the serialized data."
)
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
raise NotImplementedError()
def get_number_of_chunks(self) -> int:
return len(self.files)
def _get_size(self) -> int:
size = 0
for file in self.files:
size = size + os.path.getsize(os.path.realpath(file))
return size
def _create_cids(self, hash_codec: str) -> Sequence[CID]:
return [
self._create_cid_from_file(file, hash_codec=hash_codec)
for file in self.files
]
files: List[str]
pydantic-field
required
¶A list of strings, pointing to files containing parts of the serialized data.
type: Literal['files']
pydantic-field
¶get_chunks(self, as_files=True, symlink_ok=True)
¶Retrieve the chunks belonging to this data instance.
If 'as_file' is False, return the data as bytes. If set to 'True' store it to an arbitrary location (or use an existing one), and return the path to that file. If 'as_file' is a string, write the data (bytes) into a new file using the string as path. If 'symlink_ok' is set to True, symlinking an existing file to the value of 'as_file' is also ok, otherwise copy the content.
Source code in kiara/models/values/value.py
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
raise NotImplementedError()
get_number_of_chunks(self)
¶Source code in kiara/models/values/value.py
def get_number_of_chunks(self) -> int:
return len(self.files)
SerializedInlineJson (SerializedPreStoreChunks)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializedInlineJson(SerializedPreStoreChunks):
type: Literal["inline-json"] = "inline-json"
inline_data: Any = Field(
description="Data that will not be stored externally, but inline in the containing model. This should only contain data types that can be serialized reliably using json (scalars, etc.)."
)
_json_cache: Union[bytes, None] = PrivateAttr(default=None)
def as_json(self) -> bytes:
assert self.inline_data is not None
if self._json_cache is None:
self._json_cache = orjson.dumps(
self.inline_data,
option=orjson.OPT_NON_STR_KEYS,
)
return self._json_cache
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if as_files is False:
return [self.as_json()]
else:
raise NotImplementedError()
def get_number_of_chunks(self) -> int:
return 1
def _get_size(self) -> int:
return len(self.as_json())
def _create_cids(self, hash_codec: str) -> Sequence[CID]:
return [self._create_cid_from_chunk(self.as_json(), hash_codec=hash_codec)]
inline_data: Any
pydantic-field
¶Data that will not be stored externally, but inline in the containing model. This should only contain data types that can be serialized reliably using json (scalars, etc.).
type: Literal['inline-json']
pydantic-field
¶as_json(self)
¶Source code in kiara/models/values/value.py
def as_json(self) -> bytes:
assert self.inline_data is not None
if self._json_cache is None:
self._json_cache = orjson.dumps(
self.inline_data,
option=orjson.OPT_NON_STR_KEYS,
)
return self._json_cache
get_chunks(self, as_files=True, symlink_ok=True)
¶Retrieve the chunks belonging to this data instance.
If 'as_file' is False, return the data as bytes. If set to 'True' store it to an arbitrary location (or use an existing one), and return the path to that file. If 'as_file' is a string, write the data (bytes) into a new file using the string as path. If 'symlink_ok' is set to True, symlinking an existing file to the value of 'as_file' is also ok, otherwise copy the content.
Source code in kiara/models/values/value.py
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if as_files is False:
return [self.as_json()]
else:
raise NotImplementedError()
get_number_of_chunks(self)
¶Source code in kiara/models/values/value.py
def get_number_of_chunks(self) -> int:
return 1
SerializedListOfBytes (SerializedPreStoreChunks)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializedListOfBytes(SerializedPreStoreChunks):
type: Literal["chunks"] = "chunks"
chunks: List[bytes] = Field(description="A list of byte arrays.")
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if as_files is False:
return self.chunks
else:
if as_files is None or as_files is True or isinstance(as_files, str):
# means we write all the chunks into one file
file = None if as_files is True else as_files
path = self._store_bytes_to_file(self.chunks, file=file)
return [path]
else:
assert len(as_files) == self.get_number_of_chunks()
result = []
for idx, chunk in enumerate(self.chunks):
_file = as_files[idx]
path = self._store_bytes_to_file([chunk], file=_file)
result.append(path)
return result
def get_number_of_chunks(self) -> int:
return len(self.chunks)
def _get_size(self) -> int:
size = 0
for chunk in self.chunks:
size = size + len(chunk)
return size
def _create_cids(self, hash_codec: str) -> Sequence[CID]:
return [
self._create_cid_from_chunk(chunk, hash_codec=hash_codec)
for chunk in self.chunks
]
chunks: List[bytes]
pydantic-field
required
¶A list of byte arrays.
type: Literal['chunks']
pydantic-field
¶get_chunks(self, as_files=True, symlink_ok=True)
¶Retrieve the chunks belonging to this data instance.
If 'as_file' is False, return the data as bytes. If set to 'True' store it to an arbitrary location (or use an existing one), and return the path to that file. If 'as_file' is a string, write the data (bytes) into a new file using the string as path. If 'symlink_ok' is set to True, symlinking an existing file to the value of 'as_file' is also ok, otherwise copy the content.
Source code in kiara/models/values/value.py
def get_chunks(
self, as_files: Union[bool, str, Sequence[str]] = True, symlink_ok: bool = True
) -> Iterable[Union[str, BytesLike]]:
if as_files is False:
return self.chunks
else:
if as_files is None or as_files is True or isinstance(as_files, str):
# means we write all the chunks into one file
file = None if as_files is True else as_files
path = self._store_bytes_to_file(self.chunks, file=file)
return [path]
else:
assert len(as_files) == self.get_number_of_chunks()
result = []
for idx, chunk in enumerate(self.chunks):
_file = as_files[idx]
path = self._store_bytes_to_file([chunk], file=_file)
result.append(path)
return result
get_number_of_chunks(self)
¶Source code in kiara/models/values/value.py
def get_number_of_chunks(self) -> int:
return len(self.chunks)
SerializedPreStoreChunks (SerializedChunks)
pydantic-model
¶Source code in kiara/models/values/value.py
class SerializedPreStoreChunks(SerializedChunks):
codec: str = Field(
description="The codec used to encode the chunks in this model. Using the [multicodecs](https://github.com/multiformats/multicodec) codec table."
)
def _create_cid_from_chunk(self, chunk: bytes, hash_codec: str) -> CID:
multihash = Multihash(codec=hash_codec)
hash = multihash.digest(chunk)
return create_cid_digest(digest=hash, codec=self.codec)
def _create_cid_from_file(self, file: str, hash_codec: str) -> CID:
assert hash_codec == "sha2-256"
hash_func = hashlib.sha256
file_hash = hash_func()
CHUNK_SIZE = 65536
with open(file, "rb") as f:
fb = f.read(CHUNK_SIZE)
while len(fb) > 0:
file_hash.update(fb)
fb = f.read(CHUNK_SIZE)
wrapped = multihash.wrap(file_hash.digest(), "sha2-256")
return create_cid_digest(digest=wrapped, codec=self.codec)
codec: str
pydantic-field
required
¶The codec used to encode the chunks in this model. Using the multicodecs codec table.
UnloadableData (KiaraModel)
pydantic-model
¶A special 'marker' model, indicating that the data of value can't be loaded.
In most cases, the reason this happens is because the current kiara context is missing some value types and/or modules.
Source code in kiara/models/values/value.py
class UnloadableData(KiaraModel):
"""A special 'marker' model, indicating that the data of value can't be loaded.
In most cases, the reason this happens is because the current kiara context is missing some value types and/or modules."""
_kiara_model_id = "instance.unloadable_data"
value: Value = Field(description="A reference to the value.")
def _retrieve_id(self) -> str:
return self.value.instance_id
def _retrieve_data_to_hash(self) -> Any:
return self.value.value_id.bytes
Value (ValueDetails)
pydantic-model
¶Source code in kiara/models/values/value.py
class Value(ValueDetails):
_kiara_model_id = "instance.value"
_value_data: Any = PrivateAttr(default=SpecialValue.NOT_SET)
_serialized_data: Union[None, str, SerializedData] = PrivateAttr(default=None)
_data_retrieved: bool = PrivateAttr(default=False)
_data_registry: "DataRegistry" = PrivateAttr(default=None)
# _data_type: "DataType" = PrivateAttr(default=None)
_is_stored: bool = PrivateAttr(default=False)
_cached_properties: Union["ValueMap", None] = PrivateAttr(default=None)
_lineage: Union["ValueLineage", None] = PrivateAttr(default=None)
environment_hashes: Mapping[str, Mapping[str, str]] = Field(
description="Hashes for the environments this value was created in."
)
enviroments: Union[Mapping[str, Mapping[str, Any]], None] = Field(
description="Information about the environments this value was created in.",
default=None,
)
property_links: Mapping[str, uuid.UUID] = Field(
description="Links to values that are properties of this value.",
default_factory=dict,
)
destiny_backlinks: Mapping[uuid.UUID, str] = Field(
description="Backlinks to values that this value acts as destiny/or property for.",
default_factory=dict,
)
def add_property(
self,
value_id: Union[uuid.UUID, "Value"],
property_path: str,
add_origin_to_property_value: bool = True,
):
value = None
try:
value_temp = value
value_id = value_id.value_id # type: ignore
value = value_temp
except Exception:
# in case a Value object was provided
pass
finally:
del value_temp
if add_origin_to_property_value:
if value is None:
value = self._data_registry.get_value(value=value_id) # type: ignore
if value._is_stored:
raise Exception(
f"Can't add property to value '{self.value_id}': referenced value '{value.value_id}' already locked, so it's not possible to add the property backlink (as requested)."
)
assert value is not None
if self._is_stored:
raise Exception(
f"Can't add property to value '{self.value_id}': value already locked."
)
if property_path in self.property_links.keys():
raise Exception(
f"Can't add property to value '{self.value_id}': property '{property_path}' already set."
)
self.property_links[property_path] = value_id # type: ignore
if add_origin_to_property_value:
value.add_destiny_details(
value_id=self.value_id, destiny_alias=property_path
)
self._cached_properties = None
def add_destiny_details(self, value_id: uuid.UUID, destiny_alias: str):
if self._is_stored:
raise Exception(
f"Can't set destiny_refs to value '{self.value_id}': value already locked."
)
self.destiny_backlinks[value_id] = destiny_alias # type: ignore
@property
def is_serializable(self) -> bool:
try:
if self._serialized_data == NO_SERIALIZATION_MARKER:
return False
self.serialized_data
return True
except Exception:
pass
return False
# @property
# def data_type_class(self) -> "PythonClass":
# """Return the (Python) type of the underlying 'DataType' subclass."""
# return self.data_type_info.data_type_class
@property
def serialized_data(self) -> SerializedData:
# if not self.is_set:
# raise Exception(f"Can't retrieve serialized data: value not set.")
if self._serialized_data is not None:
if isinstance(self._serialized_data, str):
raise Exception(
f"Data type '{self.data_type_name}' does not support serializing: {self._serialized_data}"
)
return self._serialized_data
self._serialized_data = self._data_registry.retrieve_persisted_value_details(
self.value_id
)
return self._serialized_data
@property
def data(self) -> Any:
if not self.is_initialized:
raise Exception(
f"Can't retrieve data for value '{self.value_id}': value not initialized yet. This is most likely a bug."
)
return self._retrieve_data()
def _retrieve_data(self) -> Any:
if self._value_data is not SpecialValue.NOT_SET:
return self._value_data
if self.value_status in [ValueStatus.NOT_SET, ValueStatus.NONE]:
self._value_data = None
return self._value_data
elif self.value_status not in [ValueStatus.SET, ValueStatus.DEFAULT]:
raise Exception(f"Invalid internal state of value '{self.value_id}'.")
retrieved = self._data_registry.retrieve_value_data(value=self)
if retrieved is None or isinstance(retrieved, SpecialValue):
raise Exception(
f"Can't set value data, invalid data type: {type(retrieved)}"
)
self._value_data = retrieved
self._data_retrieved = True
return self._value_data
# def retrieve_load_config(self) -> Optional[LoadConfig]:
# return self._data_registry.retrieve_persisted_value_details(
# value_id=self.value_id
# )
def __repr__(self):
return f"{self.__class__.__name__}(id={self.value_id}, type={self.data_type_name}, status={self.value_status.value}, initialized={self.is_initialized} optional={self.value_schema.optional})"
def _set_registry(self, data_registry: "DataRegistry") -> None:
self._data_registry = data_registry
@property
def is_initialized(self) -> bool:
result = not self.is_set or self._data_registry is not None
return result
@property
def is_stored(self) -> bool:
return self._is_stored
@property
def data_type(self) -> "DataType":
return self.data_type_info.data_type_instance
@property
def lineage(self) -> "ValueLineage":
if self._lineage is not None:
return self._lineage
from kiara.models.values.lineage import ValueLineage
self._lineage = ValueLineage(kiara=self._data_registry._kiara, value=self)
return self._lineage
@property
def property_values(self) -> "ValueMap":
if self._cached_properties is not None:
return self._cached_properties
self._cached_properties = self._data_registry.load_values(self.property_links)
return self._cached_properties
@property
def property_names(self) -> Iterable[str]:
return self.property_links.keys()
def get_property_value(self, property_key) -> "Value":
if property_key not in self.property_links.keys():
raise Exception(
f"Value '{self.value_id}' has no property with key '{property_key}."
)
return self._data_registry.get_value(self.property_links[property_key])
def get_property_data(self, property_key: str) -> Any:
return self.get_property_value(property_key=property_key).data
def get_all_property_data(self) -> Mapping[str, Any]:
return {k: self.get_property_data(k) for k in self.property_names}
def lookup_self_aliases(self) -> Set[str]:
if not self._data_registry:
raise Exception(
f"Can't lookup aliases for value '{self.value_id}': data registry not set (yet)."
)
return self._data_registry.lookup_aliases(self)
def create_info(self) -> "ValueInfo":
if not self._data_registry:
raise Exception(
f"Can't create info object for value '{self.value_id}': data registry not set (yet)."
)
return self._data_registry.create_value_info(value=self.value_id)
def create_info_data(self, **config: Any) -> Mapping[str, Any]:
show_pedigree = config.get("show_pedigree", False)
show_lineage = config.get("show_lineage", False)
show_properties = config.get("show_properties", False)
# show_destinies = config.get("show_destinies", False)
# show_destiny_backlinks = config.get("show_destiny_backlinks", False)
# show_data = config.get("show_data_preview", False)
show_serialized = config.get("show_serialized", False)
show_env_data_hashes = config.get("show_environment_hashes", False)
show_env_data = config.get("show_environment_data", False)
ignore_fields = config.get("ignore_fields", [])
table: Dict[str, Any] = {}
if "value_id" not in ignore_fields:
table["value_id"] = self.value_id
if "aliases" not in ignore_fields:
if hasattr(self, "aliases"):
table["aliases"] = self.aliases # type: ignore
if "kiara_id" not in ignore_fields:
table["kiara_id"] = self.kiara_id
for k in sorted(self.__fields__.keys()):
if (
k
in [
"serialized",
"value_id",
"aliases",
"kiara_id",
"environments",
"environment_hashes",
]
or k in ignore_fields
):
continue
attr = getattr(self, k)
if k in ["pedigree_output_name", "pedigree"]:
continue
else:
v = attr
table[k] = v
if show_pedigree:
pedigree = getattr(self, "pedigree")
table["pedigree"] = pedigree
if pedigree == ORPHAN:
pedigree_output_name: Union[Any, None] = None
else:
pedigree_output_name = getattr(self, "pedigree_output_name")
table["pedigree_output_name"] = pedigree_output_name
if show_lineage:
table["lineage"] = self.lineage
if show_serialized:
serialized = self._data_registry.retrieve_persisted_value_details(
self.value_id
)
table["serialized"] = serialized
if show_env_data_hashes:
env_hashes = Syntax(
orjson_dumps(self.environment_hashes, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
table["environment_hashes"] = env_hashes
if show_env_data:
raise NotImplementedError()
if show_properties:
if not self.property_links:
table["properties"] = {}
else:
properties = self._data_registry.load_values(self.property_links)
table["properties"] = properties
# if hasattr(self, "destiny_links") and show_destinies:
# if not self.destiny_links: # type: ignore
# table["destinies"] = {}
# else:
# destinies = self._data_registry.load_values(self.destiny_links) # type: ignore
# table["destinies"] = destinies
#
# if show_destiny_backlinks:
# if not self.destiny_backlinks:
# table["destiny backlinks"] = {}
# else:
# destiny_items: List[Any] = []
# for v_id, alias in self.destiny_backlinks.items():
# destiny_items.append(
# f"[b]Value: [i]{v_id}[/i] (destiny alias: {alias})[/b]"
# )
# rendered = self._data_registry.pretty_print_data(
# value_id=v_id, **config
# )
# destiny_items.append(rendered)
# table["destiny backlinks"] = destiny_items
#
# if show_data:
# rendered = self._data_registry.pretty_print_data(
# self.value_id, target_type="terminal_renderable"
# )
# table["data preview"] = rendered
return table
def create_renderable(self, **render_config: Any) -> RenderableType:
from kiara.utils.output import extract_renderable
show_pedigree = render_config.get("show_pedigree", False)
show_lineage = render_config.get("show_lineage", False)
show_properties = render_config.get("show_properties", False)
show_destinies = render_config.get("show_destinies", False)
show_destiny_backlinks = render_config.get("show_destiny_backlinks", False)
show_data = render_config.get("show_data_preview", False)
show_serialized = render_config.get("show_serialized", False)
show_env_data_hashes = render_config.get("show_environment_hashes", False)
show_env_data = render_config.get("show_environment_data", False)
ignore_fields = render_config.get("ignore_fields", [])
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Key", style="i")
table.add_column("Value")
info_data = self.create_info_data(**render_config)
if "value_id" not in ignore_fields:
table.add_row("value_id", str(info_data["value_id"]))
if "aliases" not in ignore_fields:
if info_data.get("aliases", None):
aliases_str = ", ".join(info_data["aliases"]) # type: ignore
table.add_row("aliases", aliases_str)
# else:
# aliases_str = "-- n/a --"
# table.add_row("aliases", aliases_str)
if "kiara_id" not in ignore_fields:
table.add_row("kiara_id", str(info_data["kiara_id"]))
table.add_row("", "")
table.add_row("", Rule())
for k in sorted(info_data.keys()):
if (
k
in [
"serialized",
"value_id",
"aliases",
"kiara_id",
"environments",
"environment_hashes",
]
or k in ignore_fields
):
continue
attr = info_data[k]
if k in ["pedigree_output_name", "pedigree"]:
continue
elif k == "value_status":
v: RenderableType = f"[i]-- {attr.value} --[/i]"
elif k == "value_size":
v = format_size(attr)
else:
v = extract_renderable(attr)
table.add_row(k, v)
if (
show_pedigree
or show_lineage
or show_serialized
or show_properties
or show_destinies
or show_destiny_backlinks
or show_env_data_hashes
or show_env_data
):
table.add_row("", "")
table.add_row("", Rule())
table.add_row("", "")
if show_pedigree:
pedigree = info_data["pedigree"]
if pedigree == ORPHAN:
v = "[i]-- external data --[/i]"
pedigree_output_name: Union[Any, None] = None
else:
v = extract_renderable(pedigree)
pedigree_output_name = info_data["pedigree_output_name"]
row = ["pedigree", v]
table.add_row(*row)
if pedigree_output_name:
row = ["pedigree_output_name", pedigree_output_name]
table.add_row(*row)
if show_lineage:
table.add_row(
"lineage", info_data["lineage"].create_renderable(include_ids=True)
)
if show_serialized:
serialized = info_data["serialized"]
table.add_row("serialized", serialized.create_renderable())
if show_env_data_hashes:
env_hashes = Syntax(
orjson_dumps(
info_data["environment_hashes"], option=orjson.OPT_INDENT_2
),
"json",
background_color="default",
)
table.add_row("environment_hashes", env_hashes)
if show_env_data:
raise NotImplementedError()
if show_properties:
if not info_data["properties"]:
table.add_row("properties", "{}")
else:
properties = info_data["properties"]
pr = properties.create_renderable(show_header=False)
table.add_row("properties", pr)
if hasattr(self, "destiny_links") and show_destinies:
if not self.destiny_links: # type: ignore
table.add_row("destinies", "{}")
else:
destinies = self._data_registry.load_values(self.destiny_links) # type: ignore
dr = destinies.create_renderable(show_header=False)
table.add_row("destinies", dr)
if show_destiny_backlinks:
if not self.destiny_backlinks:
table.add_row("destiny backlinks", "{}")
else:
destiny_items: List[Any] = []
for v_id, alias in self.destiny_backlinks.items():
destiny_items.append(Rule())
destiny_items.append(
f"[b]Value: [i]{v_id}[/i] (destiny alias: {alias})[/b]"
)
rendered = self._data_registry.pretty_print_data(
value_id=v_id, **render_config
)
destiny_items.append(rendered)
table.add_row("destiny backlinks", Group(*destiny_items))
if show_data:
rendered = self._data_registry.pretty_print_data(
self.value_id, target_type="terminal_renderable"
)
table.add_row("", "")
table.add_row("", Rule())
table.add_row("data preview", rendered)
return table
data: Any
property
readonly
¶data_type: DataType
property
readonly
¶destiny_backlinks: Mapping[uuid.UUID, str]
pydantic-field
¶Backlinks to values that this value acts as destiny/or property for.
enviroments: Mapping[str, Mapping[str, Any]]
pydantic-field
¶Information about the environments this value was created in.
environment_hashes: Mapping[str, Mapping[str, str]]
pydantic-field
required
¶Hashes for the environments this value was created in.
is_initialized: bool
property
readonly
¶is_serializable: bool
property
readonly
¶is_stored: bool
property
readonly
¶lineage: ValueLineage
property
readonly
¶property_links: Mapping[str, uuid.UUID]
pydantic-field
¶Links to values that are properties of this value.
property_names: Iterable[str]
property
readonly
¶property_values: ValueMap
property
readonly
¶serialized_data: SerializedData
property
readonly
¶add_destiny_details(self, value_id, destiny_alias)
¶Source code in kiara/models/values/value.py
def add_destiny_details(self, value_id: uuid.UUID, destiny_alias: str):
if self._is_stored:
raise Exception(
f"Can't set destiny_refs to value '{self.value_id}': value already locked."
)
self.destiny_backlinks[value_id] = destiny_alias # type: ignore
add_property(self, value_id, property_path, add_origin_to_property_value=True)
¶Source code in kiara/models/values/value.py
def add_property(
self,
value_id: Union[uuid.UUID, "Value"],
property_path: str,
add_origin_to_property_value: bool = True,
):
value = None
try:
value_temp = value
value_id = value_id.value_id # type: ignore
value = value_temp
except Exception:
# in case a Value object was provided
pass
finally:
del value_temp
if add_origin_to_property_value:
if value is None:
value = self._data_registry.get_value(value=value_id) # type: ignore
if value._is_stored:
raise Exception(
f"Can't add property to value '{self.value_id}': referenced value '{value.value_id}' already locked, so it's not possible to add the property backlink (as requested)."
)
assert value is not None
if self._is_stored:
raise Exception(
f"Can't add property to value '{self.value_id}': value already locked."
)
if property_path in self.property_links.keys():
raise Exception(
f"Can't add property to value '{self.value_id}': property '{property_path}' already set."
)
self.property_links[property_path] = value_id # type: ignore
if add_origin_to_property_value:
value.add_destiny_details(
value_id=self.value_id, destiny_alias=property_path
)
self._cached_properties = None
create_info(self)
¶Source code in kiara/models/values/value.py
def create_info(self) -> "ValueInfo":
if not self._data_registry:
raise Exception(
f"Can't create info object for value '{self.value_id}': data registry not set (yet)."
)
return self._data_registry.create_value_info(value=self.value_id)
create_info_data(self, **config)
¶Source code in kiara/models/values/value.py
def create_info_data(self, **config: Any) -> Mapping[str, Any]:
show_pedigree = config.get("show_pedigree", False)
show_lineage = config.get("show_lineage", False)
show_properties = config.get("show_properties", False)
# show_destinies = config.get("show_destinies", False)
# show_destiny_backlinks = config.get("show_destiny_backlinks", False)
# show_data = config.get("show_data_preview", False)
show_serialized = config.get("show_serialized", False)
show_env_data_hashes = config.get("show_environment_hashes", False)
show_env_data = config.get("show_environment_data", False)
ignore_fields = config.get("ignore_fields", [])
table: Dict[str, Any] = {}
if "value_id" not in ignore_fields:
table["value_id"] = self.value_id
if "aliases" not in ignore_fields:
if hasattr(self, "aliases"):
table["aliases"] = self.aliases # type: ignore
if "kiara_id" not in ignore_fields:
table["kiara_id"] = self.kiara_id
for k in sorted(self.__fields__.keys()):
if (
k
in [
"serialized",
"value_id",
"aliases",
"kiara_id",
"environments",
"environment_hashes",
]
or k in ignore_fields
):
continue
attr = getattr(self, k)
if k in ["pedigree_output_name", "pedigree"]:
continue
else:
v = attr
table[k] = v
if show_pedigree:
pedigree = getattr(self, "pedigree")
table["pedigree"] = pedigree
if pedigree == ORPHAN:
pedigree_output_name: Union[Any, None] = None
else:
pedigree_output_name = getattr(self, "pedigree_output_name")
table["pedigree_output_name"] = pedigree_output_name
if show_lineage:
table["lineage"] = self.lineage
if show_serialized:
serialized = self._data_registry.retrieve_persisted_value_details(
self.value_id
)
table["serialized"] = serialized
if show_env_data_hashes:
env_hashes = Syntax(
orjson_dumps(self.environment_hashes, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
table["environment_hashes"] = env_hashes
if show_env_data:
raise NotImplementedError()
if show_properties:
if not self.property_links:
table["properties"] = {}
else:
properties = self._data_registry.load_values(self.property_links)
table["properties"] = properties
# if hasattr(self, "destiny_links") and show_destinies:
# if not self.destiny_links: # type: ignore
# table["destinies"] = {}
# else:
# destinies = self._data_registry.load_values(self.destiny_links) # type: ignore
# table["destinies"] = destinies
#
# if show_destiny_backlinks:
# if not self.destiny_backlinks:
# table["destiny backlinks"] = {}
# else:
# destiny_items: List[Any] = []
# for v_id, alias in self.destiny_backlinks.items():
# destiny_items.append(
# f"[b]Value: [i]{v_id}[/i] (destiny alias: {alias})[/b]"
# )
# rendered = self._data_registry.pretty_print_data(
# value_id=v_id, **config
# )
# destiny_items.append(rendered)
# table["destiny backlinks"] = destiny_items
#
# if show_data:
# rendered = self._data_registry.pretty_print_data(
# self.value_id, target_type="terminal_renderable"
# )
# table["data preview"] = rendered
return table
create_renderable(self, **render_config)
¶Source code in kiara/models/values/value.py
def create_renderable(self, **render_config: Any) -> RenderableType:
from kiara.utils.output import extract_renderable
show_pedigree = render_config.get("show_pedigree", False)
show_lineage = render_config.get("show_lineage", False)
show_properties = render_config.get("show_properties", False)
show_destinies = render_config.get("show_destinies", False)
show_destiny_backlinks = render_config.get("show_destiny_backlinks", False)
show_data = render_config.get("show_data_preview", False)
show_serialized = render_config.get("show_serialized", False)
show_env_data_hashes = render_config.get("show_environment_hashes", False)
show_env_data = render_config.get("show_environment_data", False)
ignore_fields = render_config.get("ignore_fields", [])
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("Key", style="i")
table.add_column("Value")
info_data = self.create_info_data(**render_config)
if "value_id" not in ignore_fields:
table.add_row("value_id", str(info_data["value_id"]))
if "aliases" not in ignore_fields:
if info_data.get("aliases", None):
aliases_str = ", ".join(info_data["aliases"]) # type: ignore
table.add_row("aliases", aliases_str)
# else:
# aliases_str = "-- n/a --"
# table.add_row("aliases", aliases_str)
if "kiara_id" not in ignore_fields:
table.add_row("kiara_id", str(info_data["kiara_id"]))
table.add_row("", "")
table.add_row("", Rule())
for k in sorted(info_data.keys()):
if (
k
in [
"serialized",
"value_id",
"aliases",
"kiara_id",
"environments",
"environment_hashes",
]
or k in ignore_fields
):
continue
attr = info_data[k]
if k in ["pedigree_output_name", "pedigree"]:
continue
elif k == "value_status":
v: RenderableType = f"[i]-- {attr.value} --[/i]"
elif k == "value_size":
v = format_size(attr)
else:
v = extract_renderable(attr)
table.add_row(k, v)
if (
show_pedigree
or show_lineage
or show_serialized
or show_properties
or show_destinies
or show_destiny_backlinks
or show_env_data_hashes
or show_env_data
):
table.add_row("", "")
table.add_row("", Rule())
table.add_row("", "")
if show_pedigree:
pedigree = info_data["pedigree"]
if pedigree == ORPHAN:
v = "[i]-- external data --[/i]"
pedigree_output_name: Union[Any, None] = None
else:
v = extract_renderable(pedigree)
pedigree_output_name = info_data["pedigree_output_name"]
row = ["pedigree", v]
table.add_row(*row)
if pedigree_output_name:
row = ["pedigree_output_name", pedigree_output_name]
table.add_row(*row)
if show_lineage:
table.add_row(
"lineage", info_data["lineage"].create_renderable(include_ids=True)
)
if show_serialized:
serialized = info_data["serialized"]
table.add_row("serialized", serialized.create_renderable())
if show_env_data_hashes:
env_hashes = Syntax(
orjson_dumps(
info_data["environment_hashes"], option=orjson.OPT_INDENT_2
),
"json",
background_color="default",
)
table.add_row("environment_hashes", env_hashes)
if show_env_data:
raise NotImplementedError()
if show_properties:
if not info_data["properties"]:
table.add_row("properties", "{}")
else:
properties = info_data["properties"]
pr = properties.create_renderable(show_header=False)
table.add_row("properties", pr)
if hasattr(self, "destiny_links") and show_destinies:
if not self.destiny_links: # type: ignore
table.add_row("destinies", "{}")
else:
destinies = self._data_registry.load_values(self.destiny_links) # type: ignore
dr = destinies.create_renderable(show_header=False)
table.add_row("destinies", dr)
if show_destiny_backlinks:
if not self.destiny_backlinks:
table.add_row("destiny backlinks", "{}")
else:
destiny_items: List[Any] = []
for v_id, alias in self.destiny_backlinks.items():
destiny_items.append(Rule())
destiny_items.append(
f"[b]Value: [i]{v_id}[/i] (destiny alias: {alias})[/b]"
)
rendered = self._data_registry.pretty_print_data(
value_id=v_id, **render_config
)
destiny_items.append(rendered)
table.add_row("destiny backlinks", Group(*destiny_items))
if show_data:
rendered = self._data_registry.pretty_print_data(
self.value_id, target_type="terminal_renderable"
)
table.add_row("", "")
table.add_row("", Rule())
table.add_row("data preview", rendered)
return table
get_all_property_data(self)
¶Source code in kiara/models/values/value.py
def get_all_property_data(self) -> Mapping[str, Any]:
return {k: self.get_property_data(k) for k in self.property_names}
get_property_data(self, property_key)
¶Source code in kiara/models/values/value.py
def get_property_data(self, property_key: str) -> Any:
return self.get_property_value(property_key=property_key).data
get_property_value(self, property_key)
¶Source code in kiara/models/values/value.py
def get_property_value(self, property_key) -> "Value":
if property_key not in self.property_links.keys():
raise Exception(
f"Value '{self.value_id}' has no property with key '{property_key}."
)
return self._data_registry.get_value(self.property_links[property_key])
lookup_self_aliases(self)
¶Source code in kiara/models/values/value.py
def lookup_self_aliases(self) -> Set[str]:
if not self._data_registry:
raise Exception(
f"Can't lookup aliases for value '{self.value_id}': data registry not set (yet)."
)
return self._data_registry.lookup_aliases(self)
ValueDetails (KiaraModel)
pydantic-model
¶A wrapper class that manages and retieves value data and its details.
Source code in kiara/models/values/value.py
class ValueDetails(KiaraModel):
"""A wrapper class that manages and retieves value data and its details."""
_kiara_model_id = "instance.value_details"
value_id: uuid.UUID = Field(description="The id of the value.")
kiara_id: uuid.UUID = Field(
description="The id of the kiara context this value belongs to."
)
value_schema: ValueSchema = Field(
description="The schema that was used for this Value."
)
value_status: ValueStatus = Field(description="The set/unset status of this value.")
value_size: int = Field(description="The size of this value, in bytes.")
value_hash: str = Field(description="The hash of this value.")
pedigree: ValuePedigree = Field(
description="Information about the module and inputs that went into creating this value."
)
pedigree_output_name: str = Field(
description="The output name that produced this value (using the manifest inside the pedigree)."
)
data_type_info: DataTypeInfo = Field(
description="Information about the data type this value is made of."
)
def _retrieve_id(self) -> str:
return str(self.value_id)
def _retrieve_data_to_hash(self) -> Any:
return {
"value_type": self.value_schema.type,
"value_hash": self.value_hash,
"value_size": self.value_size,
}
@property
def data_type_name(self) -> str:
return self.data_type_info.data_type_name
@property
def data_type_config(self) -> Mapping[str, Any]:
return self.data_type_info.data_type_config
@property
def is_optional(self) -> bool:
return self.value_schema.optional
@property
def is_valid(self) -> bool:
"""Check whether the current value is valid"""
if self.is_optional:
return True
else:
return self.value_status == ValueStatus.SET
@property
def is_set(self) -> bool:
return self.value_status in [ValueStatus.SET, ValueStatus.DEFAULT]
@property
def value_status_string(self) -> str:
"""Print a human readable short description of this values status."""
if self.value_status == ValueStatus.DEFAULT:
return "set (default)"
elif self.value_status == ValueStatus.SET:
return "set"
elif self.value_status == ValueStatus.NONE:
result = "no value"
elif self.value_status == ValueStatus.NOT_SET:
result = "not set"
else:
raise Exception(
f"Invalid internal status of value '{self.value_id}'. This is most likely a bug."
)
if self.is_optional:
result = f"{result} (not required)"
return result
def __repr__(self):
return f"{self.__class__.__name__}(id={self.value_id}, type={self.data_type_name}, status={self.value_status.value})"
def __str__(self):
return self.__repr__()
data_type_config: Mapping[str, Any]
property
readonly
¶data_type_info: DataTypeInfo
pydantic-field
required
¶Information about the data type this value is made of.
data_type_name: str
property
readonly
¶is_optional: bool
property
readonly
¶is_set: bool
property
readonly
¶is_valid: bool
property
readonly
¶Check whether the current value is valid
kiara_id: UUID
pydantic-field
required
¶The id of the kiara context this value belongs to.
pedigree: ValuePedigree
pydantic-field
required
¶Information about the module and inputs that went into creating this value.
pedigree_output_name: str
pydantic-field
required
¶The output name that produced this value (using the manifest inside the pedigree).
value_hash: str
pydantic-field
required
¶The hash of this value.
value_id: UUID
pydantic-field
required
¶The id of the value.
value_schema: ValueSchema
pydantic-field
required
¶The schema that was used for this Value.
value_size: int
pydantic-field
required
¶The size of this value, in bytes.
value_status: ValueStatus
pydantic-field
required
¶The set/unset status of this value.
value_status_string: str
property
readonly
¶Print a human readable short description of this values status.
ValueMap (KiaraModel, MutableMapping, Generic)
pydantic-model
¶Source code in kiara/models/values/value.py
class ValueMap(KiaraModel, MutableMapping[str, Value]): # type: ignore
values_schema: Dict[str, ValueSchema] = Field(
description="The schemas for all the values in this set."
)
@property
def field_names(self) -> Iterable[str]:
return sorted(self.values_schema.keys())
@abc.abstractmethod
def get_value_obj(self, field_name: str) -> Value:
pass
@property
def all_items_valid(self) -> bool:
for field_name in self.values_schema.keys():
item = self.get_value_obj(field_name)
if not item.is_valid:
return False
return True
def _retrieve_data_to_hash(self) -> Any:
return {
k: self.get_value_obj(k).instance_cid for k in self.values_schema.keys()
}
def check_invalid(self) -> Dict[str, str]:
"""Check whether the value set is invalid, if it is, return a description of what's wrong."""
invalid: Dict[str, str] = {}
for field_name in self.values_schema.keys():
item = self.get_value_obj(field_name)
field_schema = self.values_schema[field_name]
if not field_schema.optional:
msg: Union[str, None] = None
if not item.value_status == ValueStatus.SET:
item_schema = self.values_schema[field_name]
if item_schema.is_required():
if not item.is_set:
msg = "not set"
elif item.value_status == ValueStatus.NONE:
msg = "no value"
if msg:
invalid[field_name] = msg
return invalid
def get_value_data_for_fields(
self, *field_names: str, raise_exception_when_unset: bool = False
) -> Dict[str, Any]:
"""Return the data for a one or several fields of this ValueMap.
If a value is unset, by default 'None' is returned for it. Unless 'raise_exception_when_unset' is set to 'True',
in which case an Exception will be raised (obviously).
"""
if raise_exception_when_unset:
unset: List[str] = []
for k in field_names:
v = self.get_value_obj(k)
if not v.is_set:
if raise_exception_when_unset:
unset.append(k)
if unset:
raise Exception(
f"Can't get data for fields, one or several of the requested fields are not set yet: {', '.join(unset)}."
)
result: Dict[str, Any] = {}
for k in field_names:
v = self.get_value_obj(k)
if not v.is_set:
result[k] = None
else:
result[k] = v.data
return result
def get_value_data(
self, field_name: str, raise_exception_when_unset: bool = False
) -> Any:
return self.get_value_data_for_fields(
field_name, raise_exception_when_unset=raise_exception_when_unset
)[field_name]
def get_all_value_ids(self) -> Dict[str, uuid.UUID]:
return {k: self.get_value_obj(k).value_id for k in self.field_names}
def get_all_value_data(
self, raise_exception_when_unset: bool = False
) -> Dict[str, Any]:
return self.get_value_data_for_fields(
*self.field_names,
raise_exception_when_unset=raise_exception_when_unset,
)
def set_values(self, **values) -> None:
for k, v in values.items():
self.set_value(k, v)
def set_value(self, field_name: str, data: Any) -> None:
raise Exception(
f"The value set implementation '{self.__class__.__name__}' is read-only, and does not support the setting or changing of values."
)
def __getitem__(self, item: str) -> Value:
return self.get_value_obj(item)
def __setitem__(self, key: str, value):
raise NotImplementedError()
# self.set_value(key, value)
def __delitem__(self, key: str):
raise Exception(f"Removing items not supported: {key}")
def __iter__(self):
return iter(self.field_names)
def __len__(self):
return len(list(self.values_schema))
def __repr__(self):
return f"{self.__class__.__name__}(field_names={self.field_names})"
def __str__(self):
return self.__repr__()
def create_invalid_renderable(self, **config) -> Union[RenderableType, None]:
inv = self.check_invalid()
if not inv:
return None
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("field name", style="i")
table.add_column("details", style="b red")
for field, err in inv.items():
table.add_row(field, err)
return table
def create_renderable(self, **config: Any) -> RenderableType:
in_panel = config.get("in_panel", None)
if in_panel is None:
if is_jupyter():
in_panel = True
else:
in_panel = False
render_value_data = config.get("render_value_data", True)
field_title = config.get("field_title", "field")
value_title = config.get("value_title", "value")
show_header = config.get("show_header", True)
show_type = config.get("show_data_type", False)
table = Table(show_lines=False, show_header=show_header, box=box.SIMPLE)
table.add_column(field_title, style="b")
if show_type:
table.add_column("data_type")
table.add_column(value_title, style="i")
for field_name in self.field_names:
value = self.get_value_obj(field_name=field_name)
if render_value_data:
rendered = value._data_registry.pretty_print_data(
value_id=value.value_id, target_type="terminal_renderable", **config
)
else:
rendered = value.create_renderable(**config)
if show_type:
table.add_row(field_name, value.value_schema.type, rendered)
else:
table.add_row(field_name, rendered)
if in_panel:
return Panel(table)
else:
return table
all_items_valid: bool
property
readonly
¶field_names: Iterable[str]
property
readonly
¶values_schema: Dict[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The schemas for all the values in this set.
check_invalid(self)
¶Check whether the value set is invalid, if it is, return a description of what's wrong.
Source code in kiara/models/values/value.py
def check_invalid(self) -> Dict[str, str]:
"""Check whether the value set is invalid, if it is, return a description of what's wrong."""
invalid: Dict[str, str] = {}
for field_name in self.values_schema.keys():
item = self.get_value_obj(field_name)
field_schema = self.values_schema[field_name]
if not field_schema.optional:
msg: Union[str, None] = None
if not item.value_status == ValueStatus.SET:
item_schema = self.values_schema[field_name]
if item_schema.is_required():
if not item.is_set:
msg = "not set"
elif item.value_status == ValueStatus.NONE:
msg = "no value"
if msg:
invalid[field_name] = msg
return invalid
create_invalid_renderable(self, **config)
¶Source code in kiara/models/values/value.py
def create_invalid_renderable(self, **config) -> Union[RenderableType, None]:
inv = self.check_invalid()
if not inv:
return None
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("field name", style="i")
table.add_column("details", style="b red")
for field, err in inv.items():
table.add_row(field, err)
return table
create_renderable(self, **config)
¶Source code in kiara/models/values/value.py
def create_renderable(self, **config: Any) -> RenderableType:
in_panel = config.get("in_panel", None)
if in_panel is None:
if is_jupyter():
in_panel = True
else:
in_panel = False
render_value_data = config.get("render_value_data", True)
field_title = config.get("field_title", "field")
value_title = config.get("value_title", "value")
show_header = config.get("show_header", True)
show_type = config.get("show_data_type", False)
table = Table(show_lines=False, show_header=show_header, box=box.SIMPLE)
table.add_column(field_title, style="b")
if show_type:
table.add_column("data_type")
table.add_column(value_title, style="i")
for field_name in self.field_names:
value = self.get_value_obj(field_name=field_name)
if render_value_data:
rendered = value._data_registry.pretty_print_data(
value_id=value.value_id, target_type="terminal_renderable", **config
)
else:
rendered = value.create_renderable(**config)
if show_type:
table.add_row(field_name, value.value_schema.type, rendered)
else:
table.add_row(field_name, rendered)
if in_panel:
return Panel(table)
else:
return table
get_all_value_data(self, raise_exception_when_unset=False)
¶Source code in kiara/models/values/value.py
def get_all_value_data(
self, raise_exception_when_unset: bool = False
) -> Dict[str, Any]:
return self.get_value_data_for_fields(
*self.field_names,
raise_exception_when_unset=raise_exception_when_unset,
)
get_all_value_ids(self)
¶Source code in kiara/models/values/value.py
def get_all_value_ids(self) -> Dict[str, uuid.UUID]:
return {k: self.get_value_obj(k).value_id for k in self.field_names}
get_value_data(self, field_name, raise_exception_when_unset=False)
¶Source code in kiara/models/values/value.py
def get_value_data(
self, field_name: str, raise_exception_when_unset: bool = False
) -> Any:
return self.get_value_data_for_fields(
field_name, raise_exception_when_unset=raise_exception_when_unset
)[field_name]
get_value_data_for_fields(self, *field_names, *, raise_exception_when_unset=False)
¶Return the data for a one or several fields of this ValueMap.
If a value is unset, by default 'None' is returned for it. Unless 'raise_exception_when_unset' is set to 'True', in which case an Exception will be raised (obviously).
Source code in kiara/models/values/value.py
def get_value_data_for_fields(
self, *field_names: str, raise_exception_when_unset: bool = False
) -> Dict[str, Any]:
"""Return the data for a one or several fields of this ValueMap.
If a value is unset, by default 'None' is returned for it. Unless 'raise_exception_when_unset' is set to 'True',
in which case an Exception will be raised (obviously).
"""
if raise_exception_when_unset:
unset: List[str] = []
for k in field_names:
v = self.get_value_obj(k)
if not v.is_set:
if raise_exception_when_unset:
unset.append(k)
if unset:
raise Exception(
f"Can't get data for fields, one or several of the requested fields are not set yet: {', '.join(unset)}."
)
result: Dict[str, Any] = {}
for k in field_names:
v = self.get_value_obj(k)
if not v.is_set:
result[k] = None
else:
result[k] = v.data
return result
get_value_obj(self, field_name)
¶Source code in kiara/models/values/value.py
@abc.abstractmethod
def get_value_obj(self, field_name: str) -> Value:
pass
set_value(self, field_name, data)
¶Source code in kiara/models/values/value.py
def set_value(self, field_name: str, data: Any) -> None:
raise Exception(
f"The value set implementation '{self.__class__.__name__}' is read-only, and does not support the setting or changing of values."
)
set_values(self, **values)
¶Source code in kiara/models/values/value.py
def set_values(self, **values) -> None:
for k, v in values.items():
self.set_value(k, v)
ValueMapReadOnly (ValueMap)
pydantic-model
¶Source code in kiara/models/values/value.py
class ValueMapReadOnly(ValueMap): # type: ignore
_kiara_model_id = "instance.value_map.readonly"
@classmethod
def create_from_ids(cls, data_registry: "DataRegistry", **value_ids: uuid.UUID):
values = {k: data_registry.get_value(v) for k, v in value_ids.items()}
values_schema = {k: v.value_schema for k, v in values.items()}
return ValueMapReadOnly.construct(
value_items=values, values_schema=values_schema
)
value_items: Dict[str, Value] = Field(
description="The values contained in this set."
)
def get_value_obj(self, field_name: str) -> Value:
if field_name not in self.value_items.keys():
raise KeyError(
f"Field '{field_name}' not available in value set. Available fields: {', '.join(self.field_names)}"
)
return self.value_items[field_name]
value_items: Dict[str, kiara.models.values.value.Value]
pydantic-field
required
¶The values contained in this set.
create_from_ids(data_registry, **value_ids)
classmethod
¶Source code in kiara/models/values/value.py
@classmethod
def create_from_ids(cls, data_registry: "DataRegistry", **value_ids: uuid.UUID):
values = {k: data_registry.get_value(v) for k, v in value_ids.items()}
values_schema = {k: v.value_schema for k, v in values.items()}
return ValueMapReadOnly.construct(
value_items=values, values_schema=values_schema
)
get_value_obj(self, field_name)
¶Source code in kiara/models/values/value.py
def get_value_obj(self, field_name: str) -> Value:
if field_name not in self.value_items.keys():
raise KeyError(
f"Field '{field_name}' not available in value set. Available fields: {', '.join(self.field_names)}"
)
return self.value_items[field_name]
ValueMapWritable (ValueMap)
pydantic-model
¶Source code in kiara/models/values/value.py
class ValueMapWritable(ValueMap): # type: ignore
_kiara_model_id = "instance.value_map.writeable"
@classmethod
def create_from_schema(
cls,
kiara: "Kiara",
schema: Mapping[str, ValueSchema],
pedigree: ValuePedigree,
unique_value_ids: bool = False,
) -> "ValueMapWritable":
v = ValueMapWritable(
values_schema=dict(schema),
pedigree=pedigree,
unique_value_ids=unique_value_ids,
)
v._kiara = kiara
v._data_registry = kiara.data_registry
return v
value_items: Dict[str, Value] = Field(
description="The values contained in this set.", default_factory=dict
)
pedigree: ValuePedigree = Field(
description="The pedigree to add to all of the result values."
)
unique_value_ids: bool = Field(
description="Whether this value map always creates new value(id)s, even when a dataset with matching hash is found.",
default=True,
)
_values_uncommitted: Dict[str, Any] = PrivateAttr(default_factory=dict)
_kiara: "Kiara" = PrivateAttr(default=None)
_data_registry: "DataRegistry" = PrivateAttr(default=None)
_auto_commit: bool = PrivateAttr(default=True)
def get_value_obj(self, field_name: str) -> Value:
"""Retrieve the value object for the specified field.
This class only creates the actual value object the first time it is requested, because there is a potential
cost to assembling it, and it might not be needed ever.
"""
if field_name not in self.values_schema.keys():
raise Exception(
f"Can't set data for field '{field_name}': field not valid, valid field names: {', '.join(self.field_names)}."
)
if field_name in self.value_items.keys():
return self.value_items[field_name]
elif field_name not in self._values_uncommitted.keys():
raise Exception(
f"Can't retrieve value for field '{field_name}': value not set (yet)."
)
schema = self.values_schema[field_name]
value_data = self._values_uncommitted[field_name]
if isinstance(value_data, Value):
value = value_data
elif isinstance(value_data, uuid.UUID):
value = self._data_registry.get_value(value_data)
else:
value = self._data_registry.register_data(
data=value_data,
schema=schema,
pedigree=self.pedigree,
pedigree_output_name=field_name,
reuse_existing=not self.unique_value_ids,
)
self._values_uncommitted.pop(field_name)
self.value_items[field_name] = value
return self.value_items[field_name]
def sync_values(self):
for field_name in self.field_names:
self.get_value_obj(field_name)
invalid = self.check_invalid()
if invalid:
e = InvalidValuesException(invalid_values=invalid)
try:
raise e
except Exception:
# this is silly, I know
log_exception(e)
raise e
def set_value(self, field_name: str, data: Any) -> None:
"""Set the value for the specified field."""
if field_name not in self.field_names:
raise Exception(
f"Can't set data for field '{field_name}': field not valid, valid field names: {', '.join(self.field_names)}."
)
if self.value_items.get(field_name, False):
raise Exception(
f"Can't set data for field '{field_name}': field already committed."
)
if self._values_uncommitted.get(field_name, None) is not None:
raise Exception(
f"Can't set data for field '{field_name}': field already set."
)
self._values_uncommitted[field_name] = data
if self._auto_commit:
self.get_value_obj(field_name=field_name)
pedigree: ValuePedigree
pydantic-field
required
¶The pedigree to add to all of the result values.
unique_value_ids: bool
pydantic-field
¶Whether this value map always creates new value(id)s, even when a dataset with matching hash is found.
value_items: Dict[str, kiara.models.values.value.Value]
pydantic-field
¶The values contained in this set.
create_from_schema(kiara, schema, pedigree, unique_value_ids=False)
classmethod
¶Source code in kiara/models/values/value.py
@classmethod
def create_from_schema(
cls,
kiara: "Kiara",
schema: Mapping[str, ValueSchema],
pedigree: ValuePedigree,
unique_value_ids: bool = False,
) -> "ValueMapWritable":
v = ValueMapWritable(
values_schema=dict(schema),
pedigree=pedigree,
unique_value_ids=unique_value_ids,
)
v._kiara = kiara
v._data_registry = kiara.data_registry
return v
get_value_obj(self, field_name)
¶Retrieve the value object for the specified field.
This class only creates the actual value object the first time it is requested, because there is a potential cost to assembling it, and it might not be needed ever.
Source code in kiara/models/values/value.py
def get_value_obj(self, field_name: str) -> Value:
"""Retrieve the value object for the specified field.
This class only creates the actual value object the first time it is requested, because there is a potential
cost to assembling it, and it might not be needed ever.
"""
if field_name not in self.values_schema.keys():
raise Exception(
f"Can't set data for field '{field_name}': field not valid, valid field names: {', '.join(self.field_names)}."
)
if field_name in self.value_items.keys():
return self.value_items[field_name]
elif field_name not in self._values_uncommitted.keys():
raise Exception(
f"Can't retrieve value for field '{field_name}': value not set (yet)."
)
schema = self.values_schema[field_name]
value_data = self._values_uncommitted[field_name]
if isinstance(value_data, Value):
value = value_data
elif isinstance(value_data, uuid.UUID):
value = self._data_registry.get_value(value_data)
else:
value = self._data_registry.register_data(
data=value_data,
schema=schema,
pedigree=self.pedigree,
pedigree_output_name=field_name,
reuse_existing=not self.unique_value_ids,
)
self._values_uncommitted.pop(field_name)
self.value_items[field_name] = value
return self.value_items[field_name]
set_value(self, field_name, data)
¶Set the value for the specified field.
Source code in kiara/models/values/value.py
def set_value(self, field_name: str, data: Any) -> None:
"""Set the value for the specified field."""
if field_name not in self.field_names:
raise Exception(
f"Can't set data for field '{field_name}': field not valid, valid field names: {', '.join(self.field_names)}."
)
if self.value_items.get(field_name, False):
raise Exception(
f"Can't set data for field '{field_name}': field already committed."
)
if self._values_uncommitted.get(field_name, None) is not None:
raise Exception(
f"Can't set data for field '{field_name}': field already set."
)
self._values_uncommitted[field_name] = data
if self._auto_commit:
self.get_value_obj(field_name=field_name)
sync_values(self)
¶Source code in kiara/models/values/value.py
def sync_values(self):
for field_name in self.field_names:
self.get_value_obj(field_name)
invalid = self.check_invalid()
if invalid:
e = InvalidValuesException(invalid_values=invalid)
try:
raise e
except Exception:
# this is silly, I know
log_exception(e)
raise e
ValuePedigree (InputsManifest)
pydantic-model
¶Source code in kiara/models/values/value.py
class ValuePedigree(InputsManifest):
_kiara_model_id = "instance.value_pedigree"
kiara_id: uuid.UUID = Field(
description="The id of the kiara context a value was created in."
)
environments: Dict[str, str] = Field(
description="References to the runtime environment details a value was created in."
)
def _retrieve_data_to_hash(self) -> Any:
return {
"manifest": self.manifest_cid,
"inputs": self.inputs_cid,
"environments": self.environments,
}
def __repr__(self):
return f"ValuePedigree(module_type={self.module_type}, inputs=[{', '.join(self.inputs.keys())}], instance_id={self.instance_id})"
def __str__(self):
return self.__repr__()
value_metadata
special
¶
MetadataTypeClassesInfo (TypeInfoItemGroup)
pydantic-model
¶Source code in kiara/models/values/value_metadata/__init__.py
class MetadataTypeClassesInfo(TypeInfoItemGroup):
_kiara_model_id = "info.metadata_types"
@classmethod
def base_info_class(cls) -> Type[TypeInfo]:
return MetadataTypeInfo
type_name: Literal["value_metadata"] = "value_metadata"
item_infos: Mapping[str, MetadataTypeInfo] = Field( # type: ignore
description="The value metadata info instances for each type."
)
MetadataTypeInfo (TypeInfo)
pydantic-model
¶Source code in kiara/models/values/value_metadata/__init__.py
class MetadataTypeInfo(TypeInfo):
_kiara_model_id = "info.metadata_type"
@classmethod
def create_from_type_class(
self, type_cls: Type[ValueMetadata], kiara: "Kiara"
) -> "MetadataTypeInfo":
authors_md = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
python_class = PythonClass.from_class(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
type_name = type_cls._metadata_key # type: ignore
schema = type_cls.schema()
return MetadataTypeInfo.construct(
type_name=type_name,
documentation=doc,
authors=authors_md,
context=properties_md,
python_class=python_class,
metadata_schema=schema,
)
@classmethod
def base_class(self) -> Type[ValueMetadata]:
return ValueMetadata
@classmethod
def category_name(cls) -> str:
return "value_metadata"
metadata_schema: Dict[str, Any] = Field(
description="The (json) schema for this metadata value."
)
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
include_schema = config.get("include_schema", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
if hasattr(self, "python_class"):
table.add_row("Python class", self.python_class.create_renderable())
if include_schema:
schema = Syntax(
orjson_dumps(self.metadata_schema, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
table.add_row("metadata_schema", schema)
return table
metadata_schema: Dict[str, Any]
pydantic-field
required
¶The (json) schema for this metadata value.
base_class()
classmethod
¶Source code in kiara/models/values/value_metadata/__init__.py
@classmethod
def base_class(self) -> Type[ValueMetadata]:
return ValueMetadata
category_name()
classmethod
¶Source code in kiara/models/values/value_metadata/__init__.py
@classmethod
def category_name(cls) -> str:
return "value_metadata"
create_from_type_class(type_cls, kiara)
classmethod
¶Source code in kiara/models/values/value_metadata/__init__.py
@classmethod
def create_from_type_class(
self, type_cls: Type[ValueMetadata], kiara: "Kiara"
) -> "MetadataTypeInfo":
authors_md = AuthorsMetadataModel.from_class(type_cls)
doc = DocumentationMetadataModel.from_class_doc(type_cls)
python_class = PythonClass.from_class(type_cls)
properties_md = ContextMetadataModel.from_class(type_cls)
type_name = type_cls._metadata_key # type: ignore
schema = type_cls.schema()
return MetadataTypeInfo.construct(
type_name=type_name,
documentation=doc,
authors=authors_md,
context=properties_md,
python_class=python_class,
metadata_schema=schema,
)
create_renderable(self, **config)
¶Source code in kiara/models/values/value_metadata/__init__.py
def create_renderable(self, **config: Any) -> RenderableType:
include_doc = config.get("include_doc", True)
include_schema = config.get("include_schema", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"Documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
table.add_row("Author(s)", self.authors.create_renderable())
table.add_row("Context", self.context.create_renderable())
if hasattr(self, "python_class"):
table.add_row("Python class", self.python_class.create_renderable())
if include_schema:
schema = Syntax(
orjson_dumps(self.metadata_schema, option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
table.add_row("metadata_schema", schema)
return table
ValueMetadata (KiaraModel)
pydantic-model
¶Source code in kiara/models/values/value_metadata/__init__.py
class ValueMetadata(KiaraModel):
@classmethod
@abc.abstractmethod
def retrieve_supported_data_types(cls) -> Iterable[str]:
pass
@classmethod
@abc.abstractmethod
def create_value_metadata(
cls, value: "Value"
) -> Union["ValueMetadata", Dict[str, Any]]:
pass
# @property
# def metadata_key(self) -> str:
# return self._metadata_key # type: ignore # this is added by the kiara class loading functionality
def _retrieve_id(self) -> str:
return self._metadata_key # type: ignore
def _retrieve_data_to_hash(self) -> Any:
return {"metadata": self.dict(), "schema": self.schema_json()}
create_value_metadata(value)
classmethod
¶Source code in kiara/models/values/value_metadata/__init__.py
@classmethod
@abc.abstractmethod
def create_value_metadata(
cls, value: "Value"
) -> Union["ValueMetadata", Dict[str, Any]]:
pass
retrieve_supported_data_types()
classmethod
¶Source code in kiara/models/values/value_metadata/__init__.py
@classmethod
@abc.abstractmethod
def retrieve_supported_data_types(cls) -> Iterable[str]:
pass
included_metadata_types
special
¶
FileBundleMetadata (ValueMetadata)
pydantic-model
¶File bundle stats.
Source code in kiara/models/values/value_metadata/included_metadata_types/__init__.py
class FileBundleMetadata(ValueMetadata):
"""File bundle stats."""
_metadata_key = "file_bundle"
_kiara_model_id = "metadata.file_bundle"
@classmethod
def retrieve_supported_data_types(cls) -> Iterable[str]:
return ["file_bundle"]
@classmethod
def create_value_metadata(cls, value: "Value") -> "FileBundleMetadata":
return FileBundleMetadata.construct(file_bundle=value.data)
file_bundle: FileBundle = Field(description="The file-specific metadata.")
file_bundle: FileBundle
pydantic-field
required
¶The file-specific metadata.
create_value_metadata(value)
classmethod
¶Source code in kiara/models/values/value_metadata/included_metadata_types/__init__.py
@classmethod
def create_value_metadata(cls, value: "Value") -> "FileBundleMetadata":
return FileBundleMetadata.construct(file_bundle=value.data)
retrieve_supported_data_types()
classmethod
¶Source code in kiara/models/values/value_metadata/included_metadata_types/__init__.py
@classmethod
def retrieve_supported_data_types(cls) -> Iterable[str]:
return ["file_bundle"]
FileMetadata (ValueMetadata)
pydantic-model
¶File stats.
Source code in kiara/models/values/value_metadata/included_metadata_types/__init__.py
class FileMetadata(ValueMetadata):
"""File stats."""
_metadata_key = "file"
_kiara_model_id = "metadata.file"
@classmethod
def retrieve_supported_data_types(cls) -> Iterable[str]:
return ["file"]
@classmethod
def create_value_metadata(cls, value: "Value") -> "FileMetadata":
return FileMetadata.construct(file=value.data)
file: FileModel = Field(description="The file-specific metadata.")
file: FileModel
pydantic-field
required
¶The file-specific metadata.
create_value_metadata(value)
classmethod
¶Source code in kiara/models/values/value_metadata/included_metadata_types/__init__.py
@classmethod
def create_value_metadata(cls, value: "Value") -> "FileMetadata":
return FileMetadata.construct(file=value.data)
retrieve_supported_data_types()
classmethod
¶Source code in kiara/models/values/value_metadata/included_metadata_types/__init__.py
@classmethod
def retrieve_supported_data_types(cls) -> Iterable[str]:
return ["file"]
PythonClassMetadata (ValueMetadata)
pydantic-model
¶Python class and module information.
Source code in kiara/models/values/value_metadata/included_metadata_types/__init__.py
class PythonClassMetadata(ValueMetadata):
"""Python class and module information."""
_metadata_key = "python_class"
_kiara_model_id = "metadata.python_class"
@classmethod
def retrieve_supported_data_types(cls) -> Iterable[str]:
return ["any"]
@classmethod
def create_value_metadata(cls, value: "Value") -> "PythonClassMetadata":
return PythonClassMetadata.construct(
python_class=PythonClass.from_class(value.data.__class__)
)
# metadata_key: Literal["python_class"]
python_class: PythonClass = Field(
description="Details about the Python class that backs this value."
)
python_class: PythonClass
pydantic-field
required
¶Details about the Python class that backs this value.
create_value_metadata(value)
classmethod
¶Source code in kiara/models/values/value_metadata/included_metadata_types/__init__.py
@classmethod
def create_value_metadata(cls, value: "Value") -> "PythonClassMetadata":
return PythonClassMetadata.construct(
python_class=PythonClass.from_class(value.data.__class__)
)
retrieve_supported_data_types()
classmethod
¶Source code in kiara/models/values/value_metadata/included_metadata_types/__init__.py
@classmethod
def retrieve_supported_data_types(cls) -> Iterable[str]:
return ["any"]
value_schema
¶
ValueSchema (KiaraModel)
pydantic-model
¶The schema of a value.
The schema contains the [ValueTypeOrm][kiara.data.values.ValueTypeOrm] of a value, as well as an optional default that will be used if no user input was given (yet) for a value.
For more complex container data_types like array, tables, unions etc, data_types can also be configured with values from the type_config field.
Source code in kiara/models/values/value_schema.py
class ValueSchema(KiaraModel):
"""The schema of a value.
The schema contains the [ValueTypeOrm][kiara.data.values.ValueTypeOrm] of a value, as well as an optional default that
will be used if no user input was given (yet) for a value.
For more complex container data_types like array, tables, unions etc, data_types can also be configured with values from the ``type_config`` field.
"""
_kiara_model_id = "instance.value_schema"
class Config:
use_enum_values = True
# extra = Extra.forbid
type: str = Field(description="The type of the value.")
type_config: typing.Dict[str, typing.Any] = Field(
description="Configuration for the type, in case it's complex.",
default_factory=dict,
)
default: typing.Any = Field(
description="A default value.", default=SpecialValue.NOT_SET
)
optional: bool = Field(
description="Whether this value is required (True), or whether 'None' value is allowed (False).",
default=False,
)
is_constant: bool = Field(
description="Whether the value is a constant.", default=False
)
doc: DocumentationMetadataModel = Field(
default_factory=DocumentationMetadataModel,
description="A description for the value of this input field.",
)
@validator("doc", pre=True)
def validate_doc(cls, value):
doc = DocumentationMetadataModel.create(value)
return doc
def _retrieve_data_to_hash(self) -> typing.Any:
return {"type": self.type, "type_config": self.type_config}
def is_required(self):
if self.optional:
return False
else:
if self.default in [None, SpecialValue.NOT_SET, SpecialValue.NO_VALUE]:
return True
else:
return False
# def validate_types(self, kiara: "Kiara"):
#
# if self.type not in kiara.value_type_names:
# raise ValueError(
# f"Invalid value type '{self.type}', available data_types: {kiara.value_type_names}"
# )
def __eq__(self, other):
if not isinstance(other, ValueSchema):
return False
return (self.type, self.default) == (other.type, other.default)
def __hash__(self):
return hash((self.type, self.default))
def __repr__(self):
return f"ValueSchema(type={self.type}, default={self.default}, optional={self.optional})"
def __str__(self):
return self.__repr__()
default: Any
pydantic-field
¶A default value.
doc: DocumentationMetadataModel
pydantic-field
¶A description for the value of this input field.
is_constant: bool
pydantic-field
¶Whether the value is a constant.
optional: bool
pydantic-field
¶Whether this value is required (True), or whether 'None' value is allowed (False).
type: str
pydantic-field
required
¶The type of the value.
type_config: Dict[str, Any]
pydantic-field
¶Configuration for the type, in case it's complex.
Config
¶Source code in kiara/models/values/value_schema.py
class Config:
use_enum_values = True
# extra = Extra.forbid
is_required(self)
¶Source code in kiara/models/values/value_schema.py
def is_required(self):
if self.optional:
return False
else:
if self.default in [None, SpecialValue.NOT_SET, SpecialValue.NO_VALUE]:
return True
else:
return False
validate_doc(value)
classmethod
¶Source code in kiara/models/values/value_schema.py
@validator("doc", pre=True)
def validate_doc(cls, value):
doc = DocumentationMetadataModel.create(value)
return doc
workflow
¶
Classes¶
WorkflowDetails (KiaraModel)
pydantic-model
¶Source code in kiara/models/workflow.py
class WorkflowDetails(KiaraModel):
_kiara_model_id = "instance.workflow"
workflow_id: uuid.UUID = Field(
description="The globally unique uuid for this workflow.",
default_factory=ID_REGISTRY.generate,
)
documentation: DocumentationMetadataModel = Field(
description="A description for this workflow.",
default_factory=DocumentationMetadataModel.create,
)
authors: AuthorsMetadataModel = Field(
description="The author(s) of this workflow.",
default_factory=AuthorsMetadataModel,
)
context: ContextMetadataModel = Field(
description="Workflow context details.", default_factory=ContextMetadataModel
)
current_state: Union[str, None] = Field(
description="A reference to the current state of this workflow.", default=None
)
workflow_states: Dict[datetime.datetime, str] = Field(
description="A history of all the states of this workflow.",
default_factory=dict,
)
_kiara: Union["Kiara", None] = PrivateAttr(default=None)
# _last_update: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
@validator("documentation", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
@property
def last_state_id(self) -> Union[None, str]:
if not self.workflow_states:
return None
last_date = max(self.workflow_states.keys())
workflow_state_id = self.workflow_states[last_date]
return workflow_state_id
authors: AuthorsMetadataModel
pydantic-field
¶The author(s) of this workflow.
context: ContextMetadataModel
pydantic-field
¶Workflow context details.
current_state: str
pydantic-field
¶A reference to the current state of this workflow.
documentation: DocumentationMetadataModel
pydantic-field
¶A description for this workflow.
last_state_id: Optional[str]
property
readonly
¶workflow_id: UUID
pydantic-field
¶The globally unique uuid for this workflow.
workflow_states: Dict[datetime.datetime, str]
pydantic-field
¶A history of all the states of this workflow.
validate_doc(value)
classmethod
¶Source code in kiara/models/workflow.py
@validator("documentation", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
WorkflowGroupInfo (InfoItemGroup)
pydantic-model
¶Source code in kiara/models/workflow.py
class WorkflowGroupInfo(InfoItemGroup):
_kiara_model_id = "info.workflows"
@classmethod
def base_info_class(cls) -> Type[ItemInfo]:
return WorkflowInfo
@classmethod
def create_from_workflows(
cls,
*items: "Workflow",
group_title: Union[str, None] = None,
alias_map: Union[None, Mapping[str, uuid.UUID]] = None
) -> "WorkflowGroupInfo":
workflow_infos = {
str(w.workflow_id): WorkflowInfo.create_from_workflow(workflow=w)
for w in items
}
if alias_map is None:
alias_map = {}
workflow_group_info = cls.construct(
group_title=group_title, item_infos=workflow_infos, aliases=alias_map
)
return workflow_group_info
item_infos: Mapping[str, WorkflowInfo] = Field(
description="The workflow infos objects for each workflow."
)
aliases: Mapping[str, uuid.UUID] = Field(
description="The available aliases.", default_factory=dict
)
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(box=box.SIMPLE, show_header=True)
table.add_column("alias(es)", style="i")
table.add_column("workflow_id")
table.add_column("# steps")
table.add_column("# stages")
table.add_column("# states")
table.add_column("description")
for workflow_id, wf in self.item_infos.items():
aliases = [k for k, v in self.aliases.items() if str(v) == workflow_id]
steps = len(wf.pipeline_info.pipeline_structure.steps)
stages = len(wf.pipeline_info.pipeline_structure.processing_stages)
states = len(wf.workflow_states)
if not aliases:
alias_str = ""
else:
alias_str = ", ".join(aliases)
table.add_row(
alias_str,
workflow_id,
str(steps),
str(stages),
str(states),
wf.documentation.description,
)
return table
aliases: Mapping[str, uuid.UUID]
pydantic-field
¶The available aliases.
base_info_class()
classmethod
¶Source code in kiara/models/workflow.py
@classmethod
def base_info_class(cls) -> Type[ItemInfo]:
return WorkflowInfo
create_from_workflows(*items, *, group_title=None, alias_map=None)
classmethod
¶Source code in kiara/models/workflow.py
@classmethod
def create_from_workflows(
cls,
*items: "Workflow",
group_title: Union[str, None] = None,
alias_map: Union[None, Mapping[str, uuid.UUID]] = None
) -> "WorkflowGroupInfo":
workflow_infos = {
str(w.workflow_id): WorkflowInfo.create_from_workflow(workflow=w)
for w in items
}
if alias_map is None:
alias_map = {}
workflow_group_info = cls.construct(
group_title=group_title, item_infos=workflow_infos, aliases=alias_map
)
return workflow_group_info
create_renderable(self, **config)
¶Source code in kiara/models/workflow.py
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(box=box.SIMPLE, show_header=True)
table.add_column("alias(es)", style="i")
table.add_column("workflow_id")
table.add_column("# steps")
table.add_column("# stages")
table.add_column("# states")
table.add_column("description")
for workflow_id, wf in self.item_infos.items():
aliases = [k for k, v in self.aliases.items() if str(v) == workflow_id]
steps = len(wf.pipeline_info.pipeline_structure.steps)
stages = len(wf.pipeline_info.pipeline_structure.processing_stages)
states = len(wf.workflow_states)
if not aliases:
alias_str = ""
else:
alias_str = ", ".join(aliases)
table.add_row(
alias_str,
workflow_id,
str(steps),
str(stages),
str(states),
wf.documentation.description,
)
return table
WorkflowInfo (ItemInfo)
pydantic-model
¶Source code in kiara/models/workflow.py
class WorkflowInfo(ItemInfo):
_kiara_model_id = "info.workflow"
@classmethod
def create_from_workflow(cls, workflow: "Workflow"):
wf_info = WorkflowInfo.construct(
type_name=str(workflow.workflow_id),
workflow_details=workflow.details,
workflow_states=workflow.all_states,
pipeline_info=workflow.pipeline_info,
documentation=workflow.details.documentation,
authors=workflow.details.authors,
context=workflow.details.context,
)
return wf_info
@classmethod
def category_name(cls) -> str:
return "workflow"
workflow_details: WorkflowDetails = Field(description="The workflow details.")
workflow_states: Mapping[str, WorkflowState] = Field(
description="All states for this workflow."
)
pipeline_info: PipelineInfo = Field(
description="The current state of the workflows' pipeline."
)
def create_renderable(self, **config: Any) -> RenderableType:
in_panel = config.get("in_panel", None)
if in_panel is None:
if is_jupyter():
in_panel = True
else:
in_panel = False
include_doc = config.get("include_doc", True)
include_authors = config.get("include_authors", True)
include_context = config.get("include_context", True)
include_history = config.get("include_history", True)
include_current_state = config.get("include_current_state", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
if include_authors:
table.add_row("author(s)", self.authors.create_renderable(**config))
if include_context:
table.add_row("context", self.context.create_renderable(**config))
if include_history:
history_table = Table(show_header=False, box=box.SIMPLE)
history_table.add_column("date", style="i")
history_table.add_column("id")
for d, s_id in self.workflow_details.workflow_states.items():
history_table.add_row(str(d), s_id)
table.add_row("states", history_table)
if include_current_state:
current_state_id = (
"-- n/a --"
if not self.workflow_details.current_state
else self.workflow_details.current_state
)
table.add_row("current state id", current_state_id)
table.add_row(
"current state details", self.pipeline_info.create_renderable(**config)
)
if in_panel:
return Panel(table)
else:
return table
pipeline_info: PipelineInfo
pydantic-field
required
¶The current state of the workflows' pipeline.
workflow_details: WorkflowDetails
pydantic-field
required
¶The workflow details.
workflow_states: Mapping[str, kiara.models.workflow.WorkflowState]
pydantic-field
required
¶All states for this workflow.
category_name()
classmethod
¶Source code in kiara/models/workflow.py
@classmethod
def category_name(cls) -> str:
return "workflow"
create_from_workflow(workflow)
classmethod
¶Source code in kiara/models/workflow.py
@classmethod
def create_from_workflow(cls, workflow: "Workflow"):
wf_info = WorkflowInfo.construct(
type_name=str(workflow.workflow_id),
workflow_details=workflow.details,
workflow_states=workflow.all_states,
pipeline_info=workflow.pipeline_info,
documentation=workflow.details.documentation,
authors=workflow.details.authors,
context=workflow.details.context,
)
return wf_info
create_renderable(self, **config)
¶Source code in kiara/models/workflow.py
def create_renderable(self, **config: Any) -> RenderableType:
in_panel = config.get("in_panel", None)
if in_panel is None:
if is_jupyter():
in_panel = True
else:
in_panel = False
include_doc = config.get("include_doc", True)
include_authors = config.get("include_authors", True)
include_context = config.get("include_context", True)
include_history = config.get("include_history", True)
include_current_state = config.get("include_current_state", True)
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
if include_doc:
table.add_row(
"documentation",
Panel(self.documentation.create_renderable(), box=box.SIMPLE),
)
if include_authors:
table.add_row("author(s)", self.authors.create_renderable(**config))
if include_context:
table.add_row("context", self.context.create_renderable(**config))
if include_history:
history_table = Table(show_header=False, box=box.SIMPLE)
history_table.add_column("date", style="i")
history_table.add_column("id")
for d, s_id in self.workflow_details.workflow_states.items():
history_table.add_row(str(d), s_id)
table.add_row("states", history_table)
if include_current_state:
current_state_id = (
"-- n/a --"
if not self.workflow_details.current_state
else self.workflow_details.current_state
)
table.add_row("current state id", current_state_id)
table.add_row(
"current state details", self.pipeline_info.create_renderable(**config)
)
if in_panel:
return Panel(table)
else:
return table
WorkflowState (KiaraModel)
pydantic-model
¶Source code in kiara/models/workflow.py
class WorkflowState(KiaraModel):
@classmethod
def create_from_workflow(self, workflow: "Workflow"):
steps = list(workflow._steps.values())
inputs = dict(workflow.current_inputs)
info = PipelineInfo.create_from_pipeline(
kiara=workflow._kiara, pipeline=workflow.pipeline
)
info._kiara = workflow._kiara
ws = WorkflowState(steps=steps, inputs=inputs, pipeline_info=info)
ws._kiara = workflow._kiara
ws.pipeline_info._kiara = workflow._kiara
return ws
steps: List[PipelineStep] = Field(
description="The current steps in the workflow.", default_factory=list
)
inputs: Dict[str, uuid.UUID] = Field(
description="The current (pipeline) input values.", default_factory=dict
)
pipeline_info: PipelineInfo = Field(
description="Details about the pipeline and its state."
)
_pipeline: Union[Pipeline, None] = PrivateAttr(default=None)
_kiara: "Kiara" = PrivateAttr(default=None)
def _retrieve_data_to_hash(self) -> EncodableType:
return {
"pipeline_info": self.pipeline_info.instance_cid,
"inputs": {k: str(v) for k, v in self.inputs.items()},
}
def set_inputs(self, **inputs: uuid.UUID):
for k, v in inputs.items():
if k in self.pipeline_config.structure.pipeline_inputs_schema.keys():
self.inputs[k] = v
@property
def pipeline_config(self) -> PipelineConfig:
return self.pipeline_info.pipeline_structure.pipeline_config
@property
def pipeline_structure(self) -> PipelineStructure:
return self.pipeline_info.pipeline_structure
def create_renderable(self, **config: Any) -> RenderableType:
in_panel = config.get("in_panel", None)
if in_panel is None:
if is_jupyter():
in_panel = True
else:
in_panel = False
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
table.add_row("state id", self.instance_id)
self.pipeline_info._fill_table(table=table, config=config)
if in_panel:
return Panel(table)
else:
return table
inputs: Dict[str, uuid.UUID]
pydantic-field
¶The current (pipeline) input values.
pipeline_config: PipelineConfig
property
readonly
¶pipeline_info: PipelineInfo
pydantic-field
required
¶Details about the pipeline and its state.
pipeline_structure: PipelineStructure
property
readonly
¶steps: List[kiara.models.module.pipeline.PipelineStep]
pydantic-field
¶The current steps in the workflow.
create_from_workflow(workflow)
classmethod
¶Source code in kiara/models/workflow.py
@classmethod
def create_from_workflow(self, workflow: "Workflow"):
steps = list(workflow._steps.values())
inputs = dict(workflow.current_inputs)
info = PipelineInfo.create_from_pipeline(
kiara=workflow._kiara, pipeline=workflow.pipeline
)
info._kiara = workflow._kiara
ws = WorkflowState(steps=steps, inputs=inputs, pipeline_info=info)
ws._kiara = workflow._kiara
ws.pipeline_info._kiara = workflow._kiara
return ws
create_renderable(self, **config)
¶Source code in kiara/models/workflow.py
def create_renderable(self, **config: Any) -> RenderableType:
in_panel = config.get("in_panel", None)
if in_panel is None:
if is_jupyter():
in_panel = True
else:
in_panel = False
table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0))
table.add_column("property", style="i")
table.add_column("value")
table.add_row("state id", self.instance_id)
self.pipeline_info._fill_table(table=table, config=config)
if in_panel:
return Panel(table)
else:
return table
set_inputs(self, **inputs)
¶Source code in kiara/models/workflow.py
def set_inputs(self, **inputs: uuid.UUID):
for k, v in inputs.items():
if k in self.pipeline_config.structure.pipeline_inputs_schema.keys():
self.inputs[k] = v
modules
special
¶
DEFAULT_IDEMPOTENT_INTERNAL_MODULE_CHARACTERISTICS
¶
DEFAULT_IDEMPOTENT_MODULE_CHARACTERISTICS
¶
DEFAULT_NO_IDEMPOTENT_MODULE_CHARACTERISTICS
¶
KIARA_CONFIG
¶
ValueMapSchema
¶
ValueSetSchema
¶
log
¶
yaml
¶
Classes¶
InputOutputObject (ABC)
¶
Abstract base class for classes that define inputs and outputs schemas.
Both the 'create_inputs_schemaandcreawte_outputs_schema` methods implemented by child classes return a description of the input schema of this module.
If returning a dictionary of dictionaries, the format of the return value is as follows (items with '*' are optional):
{
"[input_field_name]: {
"type": "[type]",
"doc*": "[a description of this input]",
"optional*': [boolean whether this input is optional or required (defaults to 'False')]
"[other_input_field_name]: {
"type: ...
...
}
Source code in kiara/modules/__init__.py
class InputOutputObject(abc.ABC):
"""Abstract base class for classes that define inputs and outputs schemas.
Both the 'create_inputs_schema` and `creawte_outputs_schema` methods implemented by child classes return a description of the input schema of this module.
If returning a dictionary of dictionaries, the format of the return value is as follows (items with '*' are optional):
```
{
"[input_field_name]: {
"type": "[type]",
"doc*": "[a description of this input]",
"optional*': [boolean whether this input is optional or required (defaults to 'False')]
"[other_input_field_name]: {
"type: ...
...
}
```
"""
def __init__(
self,
alias: str,
config: KiaraModuleConfig = None,
allow_empty_inputs_schema: bool = False,
allow_empty_outputs_schema: bool = False,
):
self._alias: str = alias
self._inputs_schema: Mapping[str, ValueSchema] = None # type: ignore
self._full_inputs_schema: Mapping[str, ValueSchema] = None # type: ignore
self._outputs_schema: Mapping[str, ValueSchema] = None # type: ignore
self._constants: Mapping[str, ValueSchema] = None # type: ignore
if config is None:
config = KiaraModuleConfig()
self._config: KiaraModuleConfig = config
self._allow_empty_inputs: bool = allow_empty_inputs_schema
self._allow_empty_outputs: bool = allow_empty_outputs_schema
@property
def alias(self) -> str:
return self._alias
def input_required(self, input_name: str):
if input_name not in self._inputs_schema.keys():
raise Exception(
f"No input '{input_name}', available inputs: {', '.join(self._inputs_schema)}"
)
if not self._inputs_schema[input_name].is_required():
return False
if input_name in self.constants.keys():
return False
else:
return True
@abstractmethod
def create_inputs_schema(
self,
) -> ValueMapSchema:
"""Return the schema for this types' inputs."""
@abstractmethod
def create_outputs_schema(
self,
) -> ValueMapSchema:
"""Return the schema for this types' outputs."""
@property
def inputs_schema(self) -> Mapping[str, ValueSchema]:
"""The input schema for this module."""
if self._inputs_schema is None:
self._create_inputs_schema()
return self._inputs_schema # type: ignore
@property
def full_inputs_schema(self) -> Mapping[str, ValueSchema]:
if self._full_inputs_schema is not None:
return self._full_inputs_schema
self._full_inputs_schema = dict(self.inputs_schema)
self._full_inputs_schema.update(self._constants)
return self._full_inputs_schema
@property
def constants(self) -> Mapping[str, ValueSchema]:
if self._constants is None:
self._create_inputs_schema()
return self._constants # type: ignore
def _create_inputs_schema(self) -> None:
"""Assemble the inputs schema and assign it to the approriate instance attributes.
DEV NOTE: if anything in this method is changed, also change the method of the AutoInputsKiaraModule
in the kiara_pluginc.core_types package, since it's a carbon copy if this, except for a small change.
"""
try:
_input_schemas_data = self.create_inputs_schema()
if _input_schemas_data is None:
raise Exception(
f"Invalid inputs implementation for '{self.alias}': no inputs schema"
)
if not _input_schemas_data and not self._allow_empty_inputs:
raise Exception(
f"Invalid inputs implementation for '{self.alias}': empty inputs schema"
)
try:
_input_schemas = create_schema_dict(schema_config=_input_schemas_data)
except Exception as e:
raise Exception(f"Can't create input schemas for '{self.alias}': {e}")
defaults = self._config.defaults
constants = self._config.constants
for k, v in defaults.items():
if k not in _input_schemas.keys():
raise Exception(
f"Can't create inputs for '{self.alias}', invalid default field name '{k}'. Available field names: '{', '.join(_input_schemas.keys())}'" # type: ignore
)
for k, v in constants.items():
if k not in _input_schemas.keys():
raise Exception(
f"Can't create inputs for '{self.alias}', invalid constant field name '{k}'. Available field names: '{', '.join(_input_schemas.keys())}'" # type: ignore
)
self._inputs_schema, self._constants = overlay_constants_and_defaults(
_input_schemas, defaults=defaults, constants=constants
)
except Exception as e:
raise Exception(f"Can't create input schemas for instance '{self.alias}': {e}") # type: ignore
@property
def outputs_schema(self) -> Mapping[str, ValueSchema]:
"""The output schema for this module."""
if self._outputs_schema is not None:
return self._outputs_schema
try:
_output_schema = self.create_outputs_schema()
if _output_schema is None:
raise Exception(
f"Invalid outputs implementation for '{self.alias}': no outputs schema"
)
if not _output_schema and not self._allow_empty_outputs:
raise Exception(
f"Invalid outputs implementation for '{self.alias}': empty outputs schema"
)
try:
self._outputs_schema = create_schema_dict(schema_config=_output_schema)
except Exception as e:
raise Exception(
f"Can't create output schemas for module {self.alias}: {e}"
)
return self._outputs_schema
except Exception as e:
if is_debug():
import traceback
traceback.print_exc()
raise Exception(f"Can't create output schemas for instance of module '{self.alias}': {e}") # type: ignore
@property
def input_names(self) -> Iterable[str]:
"""A list of input field names for this module."""
return self.inputs_schema.keys()
@property
def output_names(self) -> Iterable[str]:
"""A list of output field names for this module."""
return self.outputs_schema.keys()
def augment_module_inputs(self, inputs: Mapping[str, Any]) -> Dict[str, Any]:
return augment_values(
values=inputs, schemas=self.inputs_schema, constants=self.constants
)
# def augment_outputs(self, outputs: Mapping[str, Any]) -> Dict[str, Any]:
# return augment_values(values=outputs, schemas=self.outputs_schema)
Attributes¶
alias: str
property
readonly
¶constants: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶full_inputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶input_names: Iterable[str]
property
readonly
¶A list of input field names for this module.
inputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶The input schema for this module.
output_names: Iterable[str]
property
readonly
¶A list of output field names for this module.
outputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
property
readonly
¶The output schema for this module.
Methods¶
augment_module_inputs(self, inputs)
¶Source code in kiara/modules/__init__.py
def augment_module_inputs(self, inputs: Mapping[str, Any]) -> Dict[str, Any]:
return augment_values(
values=inputs, schemas=self.inputs_schema, constants=self.constants
)
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/__init__.py
@abstractmethod
def create_inputs_schema(
self,
) -> ValueMapSchema:
"""Return the schema for this types' inputs."""
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/__init__.py
@abstractmethod
def create_outputs_schema(
self,
) -> ValueMapSchema:
"""Return the schema for this types' outputs."""
input_required(self, input_name)
¶Source code in kiara/modules/__init__.py
def input_required(self, input_name: str):
if input_name not in self._inputs_schema.keys():
raise Exception(
f"No input '{input_name}', available inputs: {', '.join(self._inputs_schema)}"
)
if not self._inputs_schema[input_name].is_required():
return False
if input_name in self.constants.keys():
return False
else:
return True
KiaraModule (InputOutputObject, Generic)
¶
The base class that every custom module in Kiara needs to inherit from.
The core of every KiaraModule is a process method, which should be a 'pure',
idempotent function that creates one or several output values from the given input(s), and its purpose is to transfor
a set of inputs into a set of outputs.
Every module can be configured. The module configuration schema can differ, but every one such configuration needs to
subclass the [KiaraModuleConfig][kiara.module_config.KiaraModuleConfig] class and set as the value to the
_config_cls attribute of the module class. This is useful, because it allows for some modules to serve a much
larger variety of use-cases than non-configurable modules would be, which would mean more code duplication because
of very simlilar, but slightly different module data_types.
Each module class (type) has a unique -- within a kiara context -- module type id which can be accessed via the
_module_type_name class attribute.
Examples:
A simple example would be an 'addition' module, with a and b configured as inputs, and z as the output field name.
An implementing class would look something like this:
TODO
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
module_config |
Union[NoneType, ~KIARA_CONFIG, Mapping[str, Any]] |
the configuation for this module |
None |
Source code in kiara/modules/__init__.py
class KiaraModule(InputOutputObject, Generic[KIARA_CONFIG]):
"""The base class that every custom module in *Kiara* needs to inherit from.
The core of every ``KiaraModule`` is a ``process`` method, which should be a 'pure',
idempotent function that creates one or several output values from the given input(s), and its purpose is to transfor
a set of inputs into a set of outputs.
Every module can be configured. The module configuration schema can differ, but every one such configuration needs to
subclass the [KiaraModuleConfig][kiara.module_config.KiaraModuleConfig] class and set as the value to the
``_config_cls`` attribute of the module class. This is useful, because it allows for some modules to serve a much
larger variety of use-cases than non-configurable modules would be, which would mean more code duplication because
of very simlilar, but slightly different module data_types.
Each module class (type) has a unique -- within a *kiara* context -- module type id which can be accessed via the
``_module_type_name`` class attribute.
Examples:
A simple example would be an 'addition' module, with ``a`` and ``b`` configured as inputs, and ``z`` as the output field name.
An implementing class would look something like this:
TODO
Arguments:
module_config: the configuation for this module
"""
# TODO: not quite sure about this generic type here, mypy doesn't seem to like it
_config_cls: Type[KIARA_CONFIG] = KiaraModuleConfig # type: ignore
@classmethod
def is_pipeline(cls) -> bool:
"""Check whether this module type is a pipeline, or not."""
return False
@classmethod
def _calculate_module_cid(
cls, module_type_config: Union[Mapping[str, Any], KIARA_CONFIG]
) -> CID:
if isinstance(module_type_config, Mapping):
module_type_config = cls._resolve_module_config(**module_type_config)
obj = {
"module_type": cls._module_type_name, # type: ignore
"module_type_config": module_type_config.dict(),
}
_, cid = compute_cid(data=obj)
return cid
@classmethod
def _resolve_module_config(cls, **config: Any) -> KIARA_CONFIG:
_config = cls._config_cls(**config) # type: ignore
return _config
def __init__(
self,
module_config: Union[None, KIARA_CONFIG, Mapping[str, Any]] = None,
):
self._id: uuid.UUID = uuid.uuid4()
if isinstance(module_config, KiaraModuleConfig):
self._config: KIARA_CONFIG = module_config # type: ignore
elif module_config is None:
self._config = self.__class__._config_cls()
elif isinstance(module_config, Mapping):
try:
self._config = self.__class__._config_cls(**module_config)
except ValidationError as ve:
raise KiaraModuleConfigException(
f"Error creating module '{id}'. {ve}",
self.__class__,
module_config,
ve,
)
else:
raise TypeError(f"Invalid type for module config: {type(module_config)}")
self._module_cid: Union[CID, None] = None
self._characteristics: Union[ModuleCharacteristics, None] = None
super().__init__(alias=self.__class__._module_type_name, config=self._config) # type: ignore
self._operation: Union[Operation, None] = None
# self._merged_input_schemas: typing.Mapping[str, ValueSchema] = None # type: ignore
self._manifest_cache: Union[None, Manifest] = None
@property
def manifest(self) -> "Manifest":
if self._manifest_cache is None:
self._manifest_cache = Manifest(
module_type=self.module_type_name,
module_config=self.config.dict(),
is_resolved=True,
)
return self._manifest_cache
@property
def module_id(self) -> uuid.UUID:
"""The id of this module."""
return self._id
@property
def module_type_name(self) -> str:
if not self._module_type_name: # type: ignore
raise Exception(
f"Module class '{self.__class__.__name__}' does not have a '_module_type_name' attribute. This is a bug."
)
return self._module_type_name # type: ignore
@property
def config(self) -> KIARA_CONFIG:
"""Retrieve the configuration object for this module.
Returns:
the module-class-specific config object
"""
return self._config
@property
def module_instance_cid(self) -> CID:
if self._module_cid is None:
self._module_cid = self.__class__._calculate_module_cid(self._config)
return self._module_cid
@property
def characteristics(self) -> ModuleCharacteristics:
if self._characteristics is not None:
return self._characteristics
self._characteristics = self._retrieve_module_characteristics()
return self._characteristics
def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
return DEFAULT_IDEMPOTENT_MODULE_CHARACTERISTICS
def get_config_value(self, key: str) -> Any:
"""Retrieve the value for a specific configuration option.
Arguments:
key: the config key
Returns:
the value for the provided key
"""
try:
return self.config.get(key)
except Exception:
raise Exception(
f"Error accessing config value '{key}' in module {self.__class__._module_type_name}." # type: ignore
)
def process_step(
self, inputs: "ValueMap", outputs: "ValueMap", job_log: JobLog
) -> None:
"""Kick off processing for a specific set of input/outputs.
This method calls the implemented [process][kiara.module.KiaraModule.process] method of the inheriting class,
as well as wrapping input/output-data related functionality.
Arguments:
inputs: the input value set
outputs: the output value set
"""
signature = inspect.signature(self.process) # type: ignore
process_inputs: Dict[str, Any] = {
"inputs": inputs,
"outputs": outputs,
}
if "job_log" in signature.parameters.keys():
process_inputs["job_log"] = job_log
try:
self.process(**process_inputs) # type: ignore
except Exception as e:
log_exception(e)
raise e
def __eq__(self, other):
if self.__class__ != other.__class__:
return False
return self.module_instance_cid == other.module_instance_cid
def __hash__(self):
return int.from_bytes(self.module_instance_cid.digest, "big")
def __repr__(self):
return f"{self.__class__.__name__}(id={self.module_id} module_type={self.module_type_name} input_names={list(self.input_names)} output_names={list(self.output_names)})"
@property
def operation(self) -> "Operation":
if self._operation is not None:
return self._operation
from kiara.models.module.operation import Operation
self._operation = Operation.create_from_module(self)
return self._operation
def create_renderable(self, **config) -> RenderableType:
return self.operation.create_renderable(**config)
Attributes¶
characteristics: ModuleCharacteristics
property
readonly
¶config: ~KIARA_CONFIG
property
readonly
¶Retrieve the configuration object for this module.
Returns:
| Type | Description |
|---|---|
~KIARA_CONFIG |
the module-class-specific config object |
manifest: Manifest
property
readonly
¶module_id: UUID
property
readonly
¶The id of this module.
module_instance_cid: CID
property
readonly
¶module_type_name: str
property
readonly
¶operation: Operation
property
readonly
¶Classes¶
_config_cls (KiaraModel)
private
pydantic-model
¶Base class that describes the configuration a [KiaraModule][kiara.module.KiaraModule] class accepts.
This is stored in the _config_cls class attribute in each KiaraModule class.
There are two config options every KiaraModule supports:
constants, anddefaults
Constants are pre-set inputs, and users can't change them and an error is thrown if they try. Defaults are default values that override the schema defaults, and those can be overwritten by users. If both a constant and a default value is set for an input field, an error is thrown.
Source code in kiara/modules/__init__.py
class KiaraModuleConfig(KiaraModel):
"""Base class that describes the configuration a [``KiaraModule``][kiara.module.KiaraModule] class accepts.
This is stored in the ``_config_cls`` class attribute in each ``KiaraModule`` class.
There are two config options every ``KiaraModule`` supports:
- ``constants``, and
- ``defaults``
Constants are pre-set inputs, and users can't change them and an error is thrown if they try. Defaults are default
values that override the schema defaults, and those can be overwritten by users. If both a constant and a default
value is set for an input field, an error is thrown.
"""
_kiara_model_id = "instance.module_config"
@classmethod
def requires_config(cls, config: Union[Mapping[str, Any], None] = None) -> bool:
"""Return whether this class can be used as-is, or requires configuration before an instance can be created."""
for field_name, field in cls.__fields__.items():
if field.required and field.default is None:
if config:
if config.get(field_name, None) is None:
return True
else:
return True
return False
_config_hash: str = PrivateAttr(default=None)
constants: Dict[str, Any] = Field(
default_factory=dict, description="Value constants for this module."
)
defaults: Dict[str, Any] = Field(
default_factory=dict, description="Value defaults for this module."
)
class Config:
extra = Extra.forbid
validate_assignment = True
def get(self, key: str) -> Any:
"""Get the value for the specified configuation key."""
if key not in self.__fields__:
raise Exception(
f"No config value '{key}' in module config class '{self.__class__.__name__}'."
)
return getattr(self, key)
def create_renderable(self, **config: Any) -> RenderableType:
my_table = Table(box=box.MINIMAL, show_header=False)
my_table.add_column("Field name", style="i")
my_table.add_column("Value")
for field in self.__fields__:
attr = getattr(self, field)
if isinstance(attr, str):
attr_str = attr
elif hasattr(attr, "create_renderable"):
attr_str = attr.create_renderable()
elif isinstance(attr, BaseModel):
attr_str = attr.json(option=orjson.orjson.OPT_INDENT_2)
else:
attr_str = str(attr)
my_table.add_row(field, attr_str)
return my_table
constants: Dict[str, Any]
pydantic-field
¶Value constants for this module.
defaults: Dict[str, Any]
pydantic-field
¶Value defaults for this module.
Config
¶create_renderable(self, **config)
¶Source code in kiara/modules/__init__.py
def create_renderable(self, **config: Any) -> RenderableType:
my_table = Table(box=box.MINIMAL, show_header=False)
my_table.add_column("Field name", style="i")
my_table.add_column("Value")
for field in self.__fields__:
attr = getattr(self, field)
if isinstance(attr, str):
attr_str = attr
elif hasattr(attr, "create_renderable"):
attr_str = attr.create_renderable()
elif isinstance(attr, BaseModel):
attr_str = attr.json(option=orjson.orjson.OPT_INDENT_2)
else:
attr_str = str(attr)
my_table.add_row(field, attr_str)
return my_table
get(self, key)
¶Get the value for the specified configuation key.
Source code in kiara/modules/__init__.py
def get(self, key: str) -> Any:
"""Get the value for the specified configuation key."""
if key not in self.__fields__:
raise Exception(
f"No config value '{key}' in module config class '{self.__class__.__name__}'."
)
return getattr(self, key)
requires_config(config=None)
classmethod
¶Return whether this class can be used as-is, or requires configuration before an instance can be created.
Source code in kiara/modules/__init__.py
@classmethod
def requires_config(cls, config: Union[Mapping[str, Any], None] = None) -> bool:
"""Return whether this class can be used as-is, or requires configuration before an instance can be created."""
for field_name, field in cls.__fields__.items():
if field.required and field.default is None:
if config:
if config.get(field_name, None) is None:
return True
else:
return True
return False
Methods¶
create_renderable(self, **config)
¶Source code in kiara/modules/__init__.py
def create_renderable(self, **config) -> RenderableType:
return self.operation.create_renderable(**config)
get_config_value(self, key)
¶Retrieve the value for a specific configuration option.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
key |
str |
the config key |
required |
Returns:
| Type | Description |
|---|---|
Any |
the value for the provided key |
Source code in kiara/modules/__init__.py
def get_config_value(self, key: str) -> Any:
"""Retrieve the value for a specific configuration option.
Arguments:
key: the config key
Returns:
the value for the provided key
"""
try:
return self.config.get(key)
except Exception:
raise Exception(
f"Error accessing config value '{key}' in module {self.__class__._module_type_name}." # type: ignore
)
is_pipeline()
classmethod
¶Check whether this module type is a pipeline, or not.
Source code in kiara/modules/__init__.py
@classmethod
def is_pipeline(cls) -> bool:
"""Check whether this module type is a pipeline, or not."""
return False
process_step(self, inputs, outputs, job_log)
¶Kick off processing for a specific set of input/outputs.
This method calls the implemented [process][kiara.module.KiaraModule.process] method of the inheriting class, as well as wrapping input/output-data related functionality.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
inputs |
ValueMap |
the input value set |
required |
outputs |
ValueMap |
the output value set |
required |
Source code in kiara/modules/__init__.py
def process_step(
self, inputs: "ValueMap", outputs: "ValueMap", job_log: JobLog
) -> None:
"""Kick off processing for a specific set of input/outputs.
This method calls the implemented [process][kiara.module.KiaraModule.process] method of the inheriting class,
as well as wrapping input/output-data related functionality.
Arguments:
inputs: the input value set
outputs: the output value set
"""
signature = inspect.signature(self.process) # type: ignore
process_inputs: Dict[str, Any] = {
"inputs": inputs,
"outputs": outputs,
}
if "job_log" in signature.parameters.keys():
process_inputs["job_log"] = job_log
try:
self.process(**process_inputs) # type: ignore
except Exception as e:
log_exception(e)
raise e
ModuleCharacteristics (BaseModel)
pydantic-model
¶
Source code in kiara/modules/__init__.py
class ModuleCharacteristics(BaseModel):
class Config:
allow_mutation = False
is_idempotent: bool = Field(
description="Whether this module is idempotent (aka always produces the same output with the same inputs.",
default=True,
)
is_internal: bool = Field(
description="Hint for frontends whether this module is used predominantly internally, and users won't need to know of its existence.",
default=False,
)
unique_result_values: bool = Field(
description="Don't re-use existing values for outputs that have matching hashes in the data store.",
default=True,
)
Attributes¶
is_idempotent: bool
pydantic-field
¶Whether this module is idempotent (aka always produces the same output with the same inputs.
is_internal: bool
pydantic-field
¶Hint for frontends whether this module is used predominantly internally, and users won't need to know of its existence.
unique_result_values: bool
pydantic-field
¶Don't re-use existing values for outputs that have matching hashes in the data store.
Config
¶Source code in kiara/modules/__init__.py
class Config:
allow_mutation = False
Modules¶
included_core_modules
special
¶
Modules¶
create_from
¶
CreateFromModule (KiaraModule)
¶Source code in kiara/modules/included_core_modules/create_from.py
class CreateFromModule(KiaraModule):
_module_type_name: str = None # type: ignore
_config_cls = CreateFromModuleConfig
@classmethod
def retrieve_supported_create_combinations(cls) -> Iterable[Mapping[str, str]]:
result = []
for attr in dir(cls):
if (
len(attr) <= 16
or not attr.startswith("create__")
or "__from__" not in attr
):
continue
tokens = attr.split("__")
if len(tokens) != 4:
continue
source_type = tokens[3]
target_type = tokens[1]
data = {
"source_type": source_type,
"target_type": target_type,
"func": attr,
}
result.append(data)
return result
def create_optional_inputs(
self, source_type: str, target_type
) -> Union[Mapping[str, Mapping[str, Any]], None]:
return None
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
source_type = self.get_config_value("source_type")
assert source_type not in ["target", "base_name"]
target_type = self.get_config_value("target_type")
optional = self.create_optional_inputs(
source_type=source_type, target_type=target_type
)
schema = {
source_type: {"type": source_type, "doc": "The type of the source value."},
}
if optional:
for field, field_schema in optional.items():
field_schema = dict(field_schema)
if field in schema.keys():
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': duplicate field '{field}'."
)
if field == source_type:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': invalid field name '{field}'."
)
optional = field_schema.get("optional", True)
if not optional:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': non-optional field '{field}' specified."
)
field_schema["optional"] = True
schema[field] = field_schema
return schema
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return {
self.get_config_value("target_type"): {
"type": self.get_config_value("target_type"),
"doc": "The result value.",
}
}
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
func_name = f"create__{target_type}__from__{source_type}"
func = getattr(self, func_name)
source_value = inputs.get_value_obj(source_type)
signature = inspect.signature(func)
if "optional" in signature.parameters:
optional: Dict[str, Value] = {}
op_schemas = {}
for field, schema in self.inputs_schema.items():
if field == source_type:
continue
optional[field] = inputs.get_value_obj(field)
op_schemas[field] = schema
result = func(
source_value=source_value,
optional=ValueMapReadOnly(
value_items=optional, values_schema=op_schemas
),
)
else:
result = func(source_value=source_value)
outputs.set_value(target_type, result)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶Source code in kiara/modules/included_core_modules/create_from.py
class CreateFromModuleConfig(KiaraModuleConfig):
source_type: str = Field(description="The value type of the source value.")
target_type: str = Field(description="The value type of the target.")
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/create_from.py
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
source_type = self.get_config_value("source_type")
assert source_type not in ["target", "base_name"]
target_type = self.get_config_value("target_type")
optional = self.create_optional_inputs(
source_type=source_type, target_type=target_type
)
schema = {
source_type: {"type": source_type, "doc": "The type of the source value."},
}
if optional:
for field, field_schema in optional.items():
field_schema = dict(field_schema)
if field in schema.keys():
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': duplicate field '{field}'."
)
if field == source_type:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': invalid field name '{field}'."
)
optional = field_schema.get("optional", True)
if not optional:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': non-optional field '{field}' specified."
)
field_schema["optional"] = True
schema[field] = field_schema
return schema
create_optional_inputs(self, source_type, target_type)
¶Source code in kiara/modules/included_core_modules/create_from.py
def create_optional_inputs(
self, source_type: str, target_type
) -> Union[Mapping[str, Mapping[str, Any]], None]:
return None
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/create_from.py
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return {
self.get_config_value("target_type"): {
"type": self.get_config_value("target_type"),
"doc": "The result value.",
}
}
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/create_from.py
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
func_name = f"create__{target_type}__from__{source_type}"
func = getattr(self, func_name)
source_value = inputs.get_value_obj(source_type)
signature = inspect.signature(func)
if "optional" in signature.parameters:
optional: Dict[str, Value] = {}
op_schemas = {}
for field, schema in self.inputs_schema.items():
if field == source_type:
continue
optional[field] = inputs.get_value_obj(field)
op_schemas[field] = schema
result = func(
source_value=source_value,
optional=ValueMapReadOnly(
value_items=optional, values_schema=op_schemas
),
)
else:
result = func(source_value=source_value)
outputs.set_value(target_type, result)
retrieve_supported_create_combinations()
classmethod
¶Source code in kiara/modules/included_core_modules/create_from.py
@classmethod
def retrieve_supported_create_combinations(cls) -> Iterable[Mapping[str, str]]:
result = []
for attr in dir(cls):
if (
len(attr) <= 16
or not attr.startswith("create__")
or "__from__" not in attr
):
continue
tokens = attr.split("__")
if len(tokens) != 4:
continue
source_type = tokens[3]
target_type = tokens[1]
data = {
"source_type": source_type,
"target_type": target_type,
"func": attr,
}
result.append(data)
return result
CreateFromModuleConfig (KiaraModuleConfig)
pydantic-model
¶Source code in kiara/modules/included_core_modules/create_from.py
class CreateFromModuleConfig(KiaraModuleConfig):
source_type: str = Field(description="The value type of the source value.")
target_type: str = Field(description="The value type of the target.")
export_as
¶
DataExportModule (KiaraModule)
¶Source code in kiara/modules/included_core_modules/export_as.py
class DataExportModule(KiaraModule):
_config_cls = DataExportModuleConfig
_module_type_name: Union[str, None] = None
@classmethod
def retrieve_supported_export_combinations(cls) -> Iterable[Mapping[str, str]]:
result = []
for attr in dir(cls):
if (
len(attr) <= 16
or not attr.startswith("export__")
or "__as__" not in attr
):
continue
tokens = attr.split("__", maxsplit=4)
if len(tokens) != 4:
continue
source_type = tokens[1]
target_profile = tokens[3]
data = {
"source_type": source_type,
"target_profile": target_profile,
"func": attr,
}
result.append(data)
return result
def create_optional_inputs(
self, source_type: str, target_profile: str
) -> Union[None, Mapping[str, Mapping[str, Any]]]:
return None
def create_inputs_schema(
self,
) -> ValueMapSchema:
source_type = self.get_config_value("source_type")
target_profile = self.get_config_value("target_profile")
inputs: Dict[str, Any] = {
source_type: {
"type": source_type,
"doc": f"A value of type '{source_type}'.",
},
"base_path": {
"type": "string",
"doc": "The directory to export the file(s) to.",
"optional": True,
},
"name": {
"type": "string",
"doc": "The (base) name of the exported file(s).",
"optional": True,
},
"export_metadata": {
"type": "boolean",
"doc": "Whether to also export the value metadata.",
"default": False,
},
}
optional = self.create_optional_inputs(
source_type=source_type, target_profile=target_profile
)
if optional:
for field, field_schema in optional.items():
field_schema = dict(field_schema)
if field in inputs.keys():
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': duplicate field '{field}'."
)
if field == source_type:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': invalid field name '{field}'."
)
optional = field_schema.get("optional", True)
if not optional:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': non-optional field '{field}' specified."
)
field_schema["optional"] = True
inputs[field] = field_schema
return inputs
def create_outputs_schema(
self,
) -> ValueMapSchema:
outputs = {
"export_details": {
"type": "dict",
"doc": "Details about the exported files/folders.",
}
}
return outputs
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
target_profile: str = self.get_config_value("target_profile")
source_type: str = self.get_config_value("source_type")
export_metadata = inputs.get_value_data("export_metadata")
source_obj = inputs.get_value_obj(source_type)
source = source_obj.data
func_name = f"export__{source_type}__as__{target_profile}"
if not hasattr(self, func_name):
raise Exception(
f"Can't export '{source_type}' value: missing function '{func_name}' in class '{self.__class__.__name__}'. Please check this modules documentation or source code to determine which source types and profiles are supported."
)
base_path = inputs.get_value_data("base_path")
if base_path is None:
base_path = os.getcwd()
name = inputs.get_value_data("name")
if not name:
name = str(source_obj.value_id)
func = getattr(self, func_name)
# TODO: check signature?
base_path = os.path.abspath(base_path)
os.makedirs(base_path, exist_ok=True)
result = func(value=source, base_path=base_path, name=name)
if isinstance(result, Mapping):
result = DataExportResult(**result)
elif isinstance(result, str):
result = DataExportResult(files=[result])
if not isinstance(result, DataExportResult):
raise KiaraProcessingException(
f"Can't export value: invalid result type '{type(result)}' from internal method. This is most likely a bug in the '{self.module_type_name}' module code."
)
if export_metadata:
metadata_file = Path(os.path.join(base_path, f"{name}.metadata"))
value_info = source_obj.create_info()
value_json = value_info.json()
metadata_file.write_text(value_json)
result.files.append(metadata_file.as_posix())
# schema = ValueSchema(type=self.get_target_value_type(), doc="Imported dataset.")
# value_lineage = ValueLineage.from_module_and_inputs(
# module=self, output_name=output_key, inputs=inputs
# )
# value: Value = self._kiara.data_registry.register_data(
# value_data=result, value_schema=schema, lineage=None
# )
outputs.set_value("export_details", result)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶Source code in kiara/modules/included_core_modules/export_as.py
class DataExportModuleConfig(KiaraModuleConfig):
target_profile: str = Field(
description="The name of the target profile. Used to distinguish different target formats for the same data type."
)
source_type: str = Field(
description="The type of the source data that is going to be exported."
)
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/export_as.py
def create_inputs_schema(
self,
) -> ValueMapSchema:
source_type = self.get_config_value("source_type")
target_profile = self.get_config_value("target_profile")
inputs: Dict[str, Any] = {
source_type: {
"type": source_type,
"doc": f"A value of type '{source_type}'.",
},
"base_path": {
"type": "string",
"doc": "The directory to export the file(s) to.",
"optional": True,
},
"name": {
"type": "string",
"doc": "The (base) name of the exported file(s).",
"optional": True,
},
"export_metadata": {
"type": "boolean",
"doc": "Whether to also export the value metadata.",
"default": False,
},
}
optional = self.create_optional_inputs(
source_type=source_type, target_profile=target_profile
)
if optional:
for field, field_schema in optional.items():
field_schema = dict(field_schema)
if field in inputs.keys():
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': duplicate field '{field}'."
)
if field == source_type:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': invalid field name '{field}'."
)
optional = field_schema.get("optional", True)
if not optional:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': non-optional field '{field}' specified."
)
field_schema["optional"] = True
inputs[field] = field_schema
return inputs
create_optional_inputs(self, source_type, target_profile)
¶Source code in kiara/modules/included_core_modules/export_as.py
def create_optional_inputs(
self, source_type: str, target_profile: str
) -> Union[None, Mapping[str, Mapping[str, Any]]]:
return None
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/export_as.py
def create_outputs_schema(
self,
) -> ValueMapSchema:
outputs = {
"export_details": {
"type": "dict",
"doc": "Details about the exported files/folders.",
}
}
return outputs
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/export_as.py
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
target_profile: str = self.get_config_value("target_profile")
source_type: str = self.get_config_value("source_type")
export_metadata = inputs.get_value_data("export_metadata")
source_obj = inputs.get_value_obj(source_type)
source = source_obj.data
func_name = f"export__{source_type}__as__{target_profile}"
if not hasattr(self, func_name):
raise Exception(
f"Can't export '{source_type}' value: missing function '{func_name}' in class '{self.__class__.__name__}'. Please check this modules documentation or source code to determine which source types and profiles are supported."
)
base_path = inputs.get_value_data("base_path")
if base_path is None:
base_path = os.getcwd()
name = inputs.get_value_data("name")
if not name:
name = str(source_obj.value_id)
func = getattr(self, func_name)
# TODO: check signature?
base_path = os.path.abspath(base_path)
os.makedirs(base_path, exist_ok=True)
result = func(value=source, base_path=base_path, name=name)
if isinstance(result, Mapping):
result = DataExportResult(**result)
elif isinstance(result, str):
result = DataExportResult(files=[result])
if not isinstance(result, DataExportResult):
raise KiaraProcessingException(
f"Can't export value: invalid result type '{type(result)}' from internal method. This is most likely a bug in the '{self.module_type_name}' module code."
)
if export_metadata:
metadata_file = Path(os.path.join(base_path, f"{name}.metadata"))
value_info = source_obj.create_info()
value_json = value_info.json()
metadata_file.write_text(value_json)
result.files.append(metadata_file.as_posix())
# schema = ValueSchema(type=self.get_target_value_type(), doc="Imported dataset.")
# value_lineage = ValueLineage.from_module_and_inputs(
# module=self, output_name=output_key, inputs=inputs
# )
# value: Value = self._kiara.data_registry.register_data(
# value_data=result, value_schema=schema, lineage=None
# )
outputs.set_value("export_details", result)
retrieve_supported_export_combinations()
classmethod
¶Source code in kiara/modules/included_core_modules/export_as.py
@classmethod
def retrieve_supported_export_combinations(cls) -> Iterable[Mapping[str, str]]:
result = []
for attr in dir(cls):
if (
len(attr) <= 16
or not attr.startswith("export__")
or "__as__" not in attr
):
continue
tokens = attr.split("__", maxsplit=4)
if len(tokens) != 4:
continue
source_type = tokens[1]
target_profile = tokens[3]
data = {
"source_type": source_type,
"target_profile": target_profile,
"func": attr,
}
result.append(data)
return result
DataExportModuleConfig (KiaraModuleConfig)
pydantic-model
¶Source code in kiara/modules/included_core_modules/export_as.py
class DataExportModuleConfig(KiaraModuleConfig):
target_profile: str = Field(
description="The name of the target profile. Used to distinguish different target formats for the same data type."
)
source_type: str = Field(
description="The type of the source data that is going to be exported."
)
DataExportResult (BaseModel)
pydantic-model
¶Source code in kiara/modules/included_core_modules/export_as.py
class DataExportResult(BaseModel):
files: List[str] = Field(description="A list of exported files.")
@validator("files", pre=True)
def validate_files(cls, value):
if isinstance(value, str):
value = [value]
# TODO: make sure file exists
return value
files: List[str]
pydantic-field
required
¶A list of exported files.
validate_files(value)
classmethod
¶Source code in kiara/modules/included_core_modules/export_as.py
@validator("files", pre=True)
def validate_files(cls, value):
if isinstance(value, str):
value = [value]
# TODO: make sure file exists
return value
filesystem
¶
DeserializeFileBundleModule (DeserializeValueModule)
¶Deserialize data to a 'file' value instance.
Source code in kiara/modules/included_core_modules/filesystem.py
class DeserializeFileBundleModule(DeserializeValueModule):
"""Deserialize data to a 'file' value instance."""
_module_type_name = "deserialize.file_bundle"
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": FileBundle}
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "file_bundle"
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "copy"
def to__python_object(self, data: SerializedData, **config: Any):
keys = list(data.get_keys())
keys.remove("__file_metadata__")
file_metadata_chunks = data.get_serialized_data("__file_metadata__")
assert file_metadata_chunks.get_number_of_chunks() == 1
file_metadata_json = list(file_metadata_chunks.get_chunks(as_files=False))
assert len(file_metadata_json) == 1
metadata = orjson.loads(file_metadata_json[0])
file_metadata = metadata["included_files"]
bundle_name = metadata["bundle_name"]
# bundle_import_time = metadata["import_time"]
sum_size = metadata["size"]
number_of_files = metadata["number_of_files"]
included_files = {}
for rel_path in keys:
chunks = data.get_serialized_data(rel_path)
assert chunks.get_number_of_chunks() == 1
files = list(chunks.get_chunks(as_files=True, symlink_ok=True))
assert len(files) == 1
file: str = files[0] # type: ignore
file_name = file_metadata[rel_path]["file_name"]
# import_time = file_metadata[rel_path]["import_time"]
fm = FileModel.load_file(source=file, file_name=file_name)
included_files[rel_path] = fm
fb = FileBundle(
included_files=included_files,
bundle_name=bundle_name,
# import_time=bundle_import_time,
number_of_files=number_of_files,
size=sum_size,
)
return fb
retrieve_serialized_value_type()
classmethod
¶Source code in kiara/modules/included_core_modules/filesystem.py
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "file_bundle"
retrieve_supported_serialization_profile()
classmethod
¶Source code in kiara/modules/included_core_modules/filesystem.py
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "copy"
retrieve_supported_target_profiles()
classmethod
¶Source code in kiara/modules/included_core_modules/filesystem.py
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": FileBundle}
to__python_object(self, data, **config)
¶Source code in kiara/modules/included_core_modules/filesystem.py
def to__python_object(self, data: SerializedData, **config: Any):
keys = list(data.get_keys())
keys.remove("__file_metadata__")
file_metadata_chunks = data.get_serialized_data("__file_metadata__")
assert file_metadata_chunks.get_number_of_chunks() == 1
file_metadata_json = list(file_metadata_chunks.get_chunks(as_files=False))
assert len(file_metadata_json) == 1
metadata = orjson.loads(file_metadata_json[0])
file_metadata = metadata["included_files"]
bundle_name = metadata["bundle_name"]
# bundle_import_time = metadata["import_time"]
sum_size = metadata["size"]
number_of_files = metadata["number_of_files"]
included_files = {}
for rel_path in keys:
chunks = data.get_serialized_data(rel_path)
assert chunks.get_number_of_chunks() == 1
files = list(chunks.get_chunks(as_files=True, symlink_ok=True))
assert len(files) == 1
file: str = files[0] # type: ignore
file_name = file_metadata[rel_path]["file_name"]
# import_time = file_metadata[rel_path]["import_time"]
fm = FileModel.load_file(source=file, file_name=file_name)
included_files[rel_path] = fm
fb = FileBundle(
included_files=included_files,
bundle_name=bundle_name,
# import_time=bundle_import_time,
number_of_files=number_of_files,
size=sum_size,
)
return fb
DeserializeFileModule (DeserializeValueModule)
¶Deserialize data to a 'file' value instance.
Source code in kiara/modules/included_core_modules/filesystem.py
class DeserializeFileModule(DeserializeValueModule):
"""Deserialize data to a 'file' value instance."""
_module_type_name = "deserialize.file"
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": FileModel}
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "file"
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "copy"
def to__python_object(self, data: SerializedData, **config: Any):
keys = list(data.get_keys())
keys.remove("__file_metadata__")
assert len(keys) == 1
file_metadata_chunks = data.get_serialized_data("__file_metadata__")
assert file_metadata_chunks.get_number_of_chunks() == 1
file_metadata_json = list(file_metadata_chunks.get_chunks(as_files=False))
assert len(file_metadata_json) == 1
file_metadata = orjson.loads(file_metadata_json[0])
chunks = data.get_serialized_data(keys[0])
assert chunks.get_number_of_chunks() == 1
files = list(chunks.get_chunks(as_files=True, symlink_ok=True))
assert len(files) == 1
file: str = files[0] # type: ignore
fm = FileModel.load_file(
source=file,
file_name=file_metadata["file_name"],
)
return fm
retrieve_serialized_value_type()
classmethod
¶Source code in kiara/modules/included_core_modules/filesystem.py
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "file"
retrieve_supported_serialization_profile()
classmethod
¶Source code in kiara/modules/included_core_modules/filesystem.py
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "copy"
retrieve_supported_target_profiles()
classmethod
¶Source code in kiara/modules/included_core_modules/filesystem.py
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": FileModel}
to__python_object(self, data, **config)
¶Source code in kiara/modules/included_core_modules/filesystem.py
def to__python_object(self, data: SerializedData, **config: Any):
keys = list(data.get_keys())
keys.remove("__file_metadata__")
assert len(keys) == 1
file_metadata_chunks = data.get_serialized_data("__file_metadata__")
assert file_metadata_chunks.get_number_of_chunks() == 1
file_metadata_json = list(file_metadata_chunks.get_chunks(as_files=False))
assert len(file_metadata_json) == 1
file_metadata = orjson.loads(file_metadata_json[0])
chunks = data.get_serialized_data(keys[0])
assert chunks.get_number_of_chunks() == 1
files = list(chunks.get_chunks(as_files=True, symlink_ok=True))
assert len(files) == 1
file: str = files[0] # type: ignore
fm = FileModel.load_file(
source=file,
file_name=file_metadata["file_name"],
)
return fm
ExportFileModule (DataExportModule)
¶Export files.
Source code in kiara/modules/included_core_modules/filesystem.py
class ExportFileModule(DataExportModule):
"""Export files."""
_module_type_name = "export.file"
def export__file__as__file(self, value: FileModel, base_path: str, name: str):
target_path = os.path.join(base_path, value.file_name)
shutil.copy2(value.path, target_path)
return {"files": target_path}
export__file__as__file(self, value, base_path, name)
¶Source code in kiara/modules/included_core_modules/filesystem.py
def export__file__as__file(self, value: FileModel, base_path: str, name: str):
target_path = os.path.join(base_path, value.file_name)
shutil.copy2(value.path, target_path)
return {"files": target_path}
ImportFileBundleModule (KiaraModule)
¶Import a folder (file_bundle) from the local filesystem.
Source code in kiara/modules/included_core_modules/filesystem.py
class ImportFileBundleModule(KiaraModule):
"""Import a folder (file_bundle) from the local filesystem."""
_module_type_name = "import.file_bundle"
def create_inputs_schema(
self,
) -> ValueMapSchema:
return {
"path": {"type": "string", "doc": "The local path of the folder to import."}
}
def create_outputs_schema(
self,
) -> ValueMapSchema:
return {
"file_bundle": {"type": "file_bundle", "doc": "The imported file bundle."}
}
def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
return DEFAULT_NO_IDEMPOTENT_MODULE_CHARACTERISTICS
def process(self, inputs: ValueMap, outputs: ValueMap):
path = inputs.get_value_data("path")
file_bundle = FileBundle.import_folder(source=path)
outputs.set_value("file_bundle", file_bundle)
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/filesystem.py
def create_inputs_schema(
self,
) -> ValueMapSchema:
return {
"path": {"type": "string", "doc": "The local path of the folder to import."}
}
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/filesystem.py
def create_outputs_schema(
self,
) -> ValueMapSchema:
return {
"file_bundle": {"type": "file_bundle", "doc": "The imported file bundle."}
}
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/filesystem.py
def process(self, inputs: ValueMap, outputs: ValueMap):
path = inputs.get_value_data("path")
file_bundle = FileBundle.import_folder(source=path)
outputs.set_value("file_bundle", file_bundle)
ImportFileModule (KiaraModule)
¶Import a file from the local filesystem.
Source code in kiara/modules/included_core_modules/filesystem.py
class ImportFileModule(KiaraModule):
"""Import a file from the local filesystem."""
_module_type_name = "import.file"
def create_inputs_schema(
self,
) -> ValueMapSchema:
return {"path": {"type": "string", "doc": "The local path to the file."}}
def create_outputs_schema(
self,
) -> ValueMapSchema:
return {"file": {"type": "file", "doc": "The loaded files."}}
def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
return ModuleCharacteristics(is_idempotent=False)
def process(self, inputs: ValueMap, outputs: ValueMap):
path = inputs.get_value_data("path")
file = FileModel.load_file(source=path)
outputs.set_value("file", file)
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/filesystem.py
def create_inputs_schema(
self,
) -> ValueMapSchema:
return {"path": {"type": "string", "doc": "The local path to the file."}}
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/filesystem.py
def create_outputs_schema(
self,
) -> ValueMapSchema:
return {"file": {"type": "file", "doc": "The loaded files."}}
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/filesystem.py
def process(self, inputs: ValueMap, outputs: ValueMap):
path = inputs.get_value_data("path")
file = FileModel.load_file(source=path)
outputs.set_value("file", file)
PickFileModule (KiaraModule)
¶Pick a single file from a file_bundle value.
Source code in kiara/modules/included_core_modules/filesystem.py
class PickFileModule(KiaraModule):
"""Pick a single file from a file_bundle value."""
_module_type_name = "file_bundle.pick_file"
def create_inputs_schema(
self,
) -> ValueMapSchema:
return {
"file_bundle": {"type": "file_bundle", "doc": "The file bundle."},
"path": {"type": "string", "doc": "The relative path of the file to pick."},
}
def create_outputs_schema(
self,
) -> ValueMapSchema:
return {"file": {"type": "file", "doc": "The file."}}
def process(self, inputs: ValueMap, outputs: ValueMap):
file_bundle: FileBundle = inputs.get_value_data("file_bundle")
path: str = inputs.get_value_data("path")
if path not in file_bundle.included_files.keys():
raise KiaraProcessingException(
f"Can't pick file '{path}' from file bundle: file not available."
)
file: FileModel = file_bundle.included_files[path]
outputs.set_value("file", file)
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/filesystem.py
def create_inputs_schema(
self,
) -> ValueMapSchema:
return {
"file_bundle": {"type": "file_bundle", "doc": "The file bundle."},
"path": {"type": "string", "doc": "The relative path of the file to pick."},
}
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/filesystem.py
def create_outputs_schema(
self,
) -> ValueMapSchema:
return {"file": {"type": "file", "doc": "The file."}}
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/filesystem.py
def process(self, inputs: ValueMap, outputs: ValueMap):
file_bundle: FileBundle = inputs.get_value_data("file_bundle")
path: str = inputs.get_value_data("path")
if path not in file_bundle.included_files.keys():
raise KiaraProcessingException(
f"Can't pick file '{path}' from file bundle: file not available."
)
file: FileModel = file_bundle.included_files[path]
outputs.set_value("file", file)
filter
¶
FilterModule (KiaraModule)
¶Source code in kiara/modules/included_core_modules/filter.py
class FilterModule(KiaraModule):
_module_type_name: Union[str, None] = None
@classmethod
def get_supported_filters(cls) -> List[str]:
result = []
for attr in dir(cls):
if len(attr) <= 8 or not attr.startswith("filter__"):
continue
filter_name = attr[8:]
result.append(filter_name)
if not result and is_develop():
from rich.table import Table
from kiara.models.python_class import PythonClass
pcls = PythonClass.from_class(cls)
tbl = Table.grid()
tbl.add_column("key", style="i")
tbl.add_column("value")
tbl.add_row(
"details",
"Module class inherits from the 'FilterModule' class, but doesn't implement any methods that start with 'filter__.",
)
tbl.add_row("reference", "TODO")
tbl.add_row("python class", pcls)
log_dev_message(tbl)
return result
@classmethod
@abstractmethod
def retrieve_supported_type(cls) -> Union[Dict[str, Any], str]:
pass
@classmethod
def get_supported_type(cls) -> Dict[str, Any]:
data = cls.retrieve_supported_type()
if isinstance(data, str):
data = {"type": data, "type_config": {}}
else:
# TODO: more validation?
assert "type" in data.keys()
if "type_config" not in data.keys():
data["type_config"] = {}
return data
_config_cls = FilterModuleConfig
def create_filter_inputs(self, filter_name: str) -> Union[None, ValueMapSchema]:
return None
def create_inputs_schema(
self,
) -> ValueMapSchema:
filter_name = self.get_config_value("filter_name")
data_type_data = self.get_supported_type()
data_type = data_type_data["type"]
data_type_config = data_type_data["type_config"]
inputs: Dict[str, Any] = {
"value": {
"type": data_type,
"type_config": data_type_config,
"doc": f"A value of type '{data_type}'.",
},
}
filter_inputs = self.create_filter_inputs(filter_name=filter_name)
if filter_inputs:
for field, field_schema in filter_inputs.items():
field_schema = dict(field_schema)
if field in inputs.keys():
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': duplicate field '{field}'."
)
filter_inputs_optional = field_schema.get("optional", False)
filter_inputs_default = field_schema.get("default", None)
if not filter_inputs_optional and filter_inputs_default is None:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': non-optional field '{field}' specified."
)
field_schema["optional"] = True
inputs[field] = field_schema
return inputs
def create_outputs_schema(
self,
) -> ValueMapSchema:
data_type_data = self.get_supported_type()
data_type = data_type_data["type"]
data_type_config = data_type_data["type_config"]
outputs = {
"value": {
"type": data_type,
"type_config": data_type_config,
"doc": "The filtered value.",
}
}
return outputs
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
filter_name: str = self.get_config_value("filter_name")
data_type_data = self.__class__.get_supported_type()
data_type = data_type_data["type"]
# data_type_config = data_type_data["type_config"]
# TODO: ensure value is of the right type?
source_obj = inputs.get_value_obj("value")
func_name = f"filter__{filter_name}"
if not hasattr(self, func_name):
raise Exception(
f"Can't apply filter '{filter_name}': missing function '{func_name}' in class '{self.__class__.__name__}'. Please check this modules documentation or source code to determine which filters are supported."
)
func = getattr(self, func_name)
# TODO: check signature?
filter_inputs = {}
for k, v in inputs.items():
if k == data_type:
continue
filter_inputs[k] = v.data
result = func(value=source_obj, filter_inputs=filter_inputs)
if result is None:
outputs.set_value("value", source_obj)
else:
outputs.set_value("value", result)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶create_filter_inputs(self, filter_name)
¶Source code in kiara/modules/included_core_modules/filter.py
def create_filter_inputs(self, filter_name: str) -> Union[None, ValueMapSchema]:
return None
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/filter.py
def create_inputs_schema(
self,
) -> ValueMapSchema:
filter_name = self.get_config_value("filter_name")
data_type_data = self.get_supported_type()
data_type = data_type_data["type"]
data_type_config = data_type_data["type_config"]
inputs: Dict[str, Any] = {
"value": {
"type": data_type,
"type_config": data_type_config,
"doc": f"A value of type '{data_type}'.",
},
}
filter_inputs = self.create_filter_inputs(filter_name=filter_name)
if filter_inputs:
for field, field_schema in filter_inputs.items():
field_schema = dict(field_schema)
if field in inputs.keys():
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': duplicate field '{field}'."
)
filter_inputs_optional = field_schema.get("optional", False)
filter_inputs_default = field_schema.get("default", None)
if not filter_inputs_optional and filter_inputs_default is None:
raise Exception(
f"Can't create inputs schema for '{self.module_type_name}': non-optional field '{field}' specified."
)
field_schema["optional"] = True
inputs[field] = field_schema
return inputs
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/filter.py
def create_outputs_schema(
self,
) -> ValueMapSchema:
data_type_data = self.get_supported_type()
data_type = data_type_data["type"]
data_type_config = data_type_data["type_config"]
outputs = {
"value": {
"type": data_type,
"type_config": data_type_config,
"doc": "The filtered value.",
}
}
return outputs
get_supported_filters()
classmethod
¶Source code in kiara/modules/included_core_modules/filter.py
@classmethod
def get_supported_filters(cls) -> List[str]:
result = []
for attr in dir(cls):
if len(attr) <= 8 or not attr.startswith("filter__"):
continue
filter_name = attr[8:]
result.append(filter_name)
if not result and is_develop():
from rich.table import Table
from kiara.models.python_class import PythonClass
pcls = PythonClass.from_class(cls)
tbl = Table.grid()
tbl.add_column("key", style="i")
tbl.add_column("value")
tbl.add_row(
"details",
"Module class inherits from the 'FilterModule' class, but doesn't implement any methods that start with 'filter__.",
)
tbl.add_row("reference", "TODO")
tbl.add_row("python class", pcls)
log_dev_message(tbl)
return result
get_supported_type()
classmethod
¶Source code in kiara/modules/included_core_modules/filter.py
@classmethod
def get_supported_type(cls) -> Dict[str, Any]:
data = cls.retrieve_supported_type()
if isinstance(data, str):
data = {"type": data, "type_config": {}}
else:
# TODO: more validation?
assert "type" in data.keys()
if "type_config" not in data.keys():
data["type_config"] = {}
return data
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/filter.py
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
filter_name: str = self.get_config_value("filter_name")
data_type_data = self.__class__.get_supported_type()
data_type = data_type_data["type"]
# data_type_config = data_type_data["type_config"]
# TODO: ensure value is of the right type?
source_obj = inputs.get_value_obj("value")
func_name = f"filter__{filter_name}"
if not hasattr(self, func_name):
raise Exception(
f"Can't apply filter '{filter_name}': missing function '{func_name}' in class '{self.__class__.__name__}'. Please check this modules documentation or source code to determine which filters are supported."
)
func = getattr(self, func_name)
# TODO: check signature?
filter_inputs = {}
for k, v in inputs.items():
if k == data_type:
continue
filter_inputs[k] = v.data
result = func(value=source_obj, filter_inputs=filter_inputs)
if result is None:
outputs.set_value("value", source_obj)
else:
outputs.set_value("value", result)
retrieve_supported_type()
classmethod
¶Source code in kiara/modules/included_core_modules/filter.py
@classmethod
@abstractmethod
def retrieve_supported_type(cls) -> Union[Dict[str, Any], str]:
pass
FilterModuleConfig (KiaraModuleConfig)
pydantic-model
¶metadata
¶
ExtractMetadataModule (KiaraModule)
¶Base class to use when writing a module to extract metadata from a file.
It's possible to use any arbitrary kiara module for this purpose, but sub-classing this makes it easier.
Source code in kiara/modules/included_core_modules/metadata.py
class ExtractMetadataModule(KiaraModule):
"""Base class to use when writing a module to extract metadata from a file.
It's possible to use any arbitrary *kiara* module for this purpose, but sub-classing this makes it easier.
"""
_config_cls = MetadataModuleConfig
_module_type_name: str = "value.extract_metadata"
def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
return ModuleCharacteristics(
is_idempotent=True, is_internal=True, unique_result_values=True
)
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
data_type_name = self.get_config_value("data_type")
inputs = {
"value": {
"type": data_type_name,
"doc": f"A value of type '{data_type_name}'",
"optional": False,
}
}
return inputs
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
kiara_model_id: str = self.get_config_value("kiara_model_id")
# TODO: check it's subclassing the right class
outputs = {
"value_metadata": {
"type": "internal_model",
"type_config": {"kiara_model_id": kiara_model_id},
"doc": "The metadata for the provided value.",
}
}
return outputs
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
value = inputs.get_value_obj("value")
kiara_model_id: str = self.get_config_value("kiara_model_id")
model_registry = ModelRegistry.instance()
metadata_model_cls: Type[ValueMetadata] = model_registry.get_model_cls(kiara_model_id=kiara_model_id, required_subclass=ValueMetadata) # type: ignore
metadata = metadata_model_cls.create_value_metadata(value=value)
if not isinstance(metadata, metadata_model_cls):
raise KiaraProcessingException(
f"Invalid metadata model result, should be class '{metadata_model_cls.__name__}', but is: {metadata.__class__.__name__}. This is most likely a bug."
)
outputs.set_value("value_metadata", metadata)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶Source code in kiara/modules/included_core_modules/metadata.py
class MetadataModuleConfig(KiaraModuleConfig):
data_type: str = Field(description="The data type this module will be used for.")
kiara_model_id: str = Field(description="The id of the kiara (metadata) model.")
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/metadata.py
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
data_type_name = self.get_config_value("data_type")
inputs = {
"value": {
"type": data_type_name,
"doc": f"A value of type '{data_type_name}'",
"optional": False,
}
}
return inputs
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/metadata.py
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
kiara_model_id: str = self.get_config_value("kiara_model_id")
# TODO: check it's subclassing the right class
outputs = {
"value_metadata": {
"type": "internal_model",
"type_config": {"kiara_model_id": kiara_model_id},
"doc": "The metadata for the provided value.",
}
}
return outputs
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/metadata.py
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
value = inputs.get_value_obj("value")
kiara_model_id: str = self.get_config_value("kiara_model_id")
model_registry = ModelRegistry.instance()
metadata_model_cls: Type[ValueMetadata] = model_registry.get_model_cls(kiara_model_id=kiara_model_id, required_subclass=ValueMetadata) # type: ignore
metadata = metadata_model_cls.create_value_metadata(value=value)
if not isinstance(metadata, metadata_model_cls):
raise KiaraProcessingException(
f"Invalid metadata model result, should be class '{metadata_model_cls.__name__}', but is: {metadata.__class__.__name__}. This is most likely a bug."
)
outputs.set_value("value_metadata", metadata)
MetadataModuleConfig (KiaraModuleConfig)
pydantic-model
¶Source code in kiara/modules/included_core_modules/metadata.py
class MetadataModuleConfig(KiaraModuleConfig):
data_type: str = Field(description="The data type this module will be used for.")
kiara_model_id: str = Field(description="The id of the kiara (metadata) model.")
pipeline
¶
PipelineModule (KiaraModule)
¶A utility module to run multiple connected inner-modules and present it as its own entity.
Source code in kiara/modules/included_core_modules/pipeline.py
class PipelineModule(KiaraModule):
"""A utility module to run multiple connected inner-modules and present it as its own entity."""
_config_cls = PipelineConfig
_module_type_name = "pipeline"
def __init__(
self,
module_config: Union[None, KIARA_CONFIG, Mapping[str, Any]] = None,
):
self._job_registry: Union[JobRegistry, None] = None
super().__init__(module_config=module_config)
@classmethod
def is_pipeline(cls) -> bool:
return True
def _set_job_registry(self, job_registry: "JobRegistry"):
self._job_registry = job_registry
@property
def operation(self) -> "Operation":
if self._operation is not None:
return self._operation
from kiara.models.module.operation import Operation
self._operation = Operation.create_from_module(self, doc=self.config.doc)
return self._operation
def create_inputs_schema(
self,
) -> ValueMapSchema:
pipeline_structure: PipelineStructure = self.config.structure
inputs_schema = pipeline_structure.pipeline_inputs_schema
return inputs_schema
def create_outputs_schema(
self,
) -> ValueMapSchema:
pipeline_structure: PipelineStructure = self.config.structure
return pipeline_structure.pipeline_outputs_schema
def process(self, inputs: ValueMap, outputs: ValueMapWritable, job_log: JobLog):
pipeline_structure: PipelineStructure = self.config.structure
pipeline = Pipeline(structure=pipeline_structure, kiara=outputs._kiara)
assert self._job_registry is not None
controller = SinglePipelineBatchController(
pipeline=pipeline, job_registry=self._job_registry
)
pipeline.set_pipeline_inputs(inputs=inputs)
controller.process_pipeline()
# TODO: resolve values first?
outputs.set_values(**pipeline.get_current_pipeline_outputs())
operation: Operation
property
readonly
¶
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶A class to hold the configuration for a [PipelineModule][kiara.pipeline.module.PipelineModule].
If you want to control the pipeline input and output names, you need to have to provide a map that uses the autogenerated field name ([step_id]__[alias] -- 2 underscores!!) as key, and the desired field name as value. The reason that schema for the autogenerated field names exist is that it's hard to ensure the uniqueness of each field; some steps can have the same input field names, but will need different input values. In some cases, some inputs of different steps need the same input. Those sorts of things. So, to make sure that we always use the right values, I chose to implement a conservative default approach, accepting that in some cases the user will be prompted for duplicate inputs for the same value.
To remedy that, the pipeline creator has the option to manually specify a mapping to rename some or all of the input/output fields.
Further, because in a lot of cases there won't be any overlapping fields, the creator can specify auto,
in which case Kiara will automatically create a mapping that tries to map autogenerated field names
to the shortest possible names for each case.
Examples:
Configuration for a pipeline module that functions as a nand logic gate (in Python):
and_step = PipelineStepConfig(module_type="and", step_id="and")
not_step = PipelineStepConfig(module_type="not", step_id="not", input_links={"a": ["and.y"]}
nand_p_conf = PipelineConfig(doc="Returns 'False' if both inputs are 'True'.",
steps=[and_step, not_step],
input_aliases={
"and__a": "a",
"and__b": "b"
},
output_aliases={
"not__y": "y"
}}
Or, the same thing in json:
{
"module_type_name": "nand",
"doc": "Returns 'False' if both inputs are 'True'.",
"steps": [
{
"module_type": "and",
"step_id": "and"
},
{
"module_type": "not",
"step_id": "not",
"input_links": {
"a": "and.y"
}
}
],
"input_aliases": {
"and__a": "a",
"and__b": "b"
},
"output_aliases": {
"not__y": "y"
}
}
Source code in kiara/modules/included_core_modules/pipeline.py
class PipelineConfig(KiaraModuleConfig):
"""A class to hold the configuration for a [PipelineModule][kiara.pipeline.module.PipelineModule].
If you want to control the pipeline input and output names, you need to have to provide a map that uses the
autogenerated field name ([step_id]__[alias] -- 2 underscores!!) as key, and the desired field name
as value. The reason that schema for the autogenerated field names exist is that it's hard to ensure
the uniqueness of each field; some steps can have the same input field names, but will need different input
values. In some cases, some inputs of different steps need the same input. Those sorts of things.
So, to make sure that we always use the right values, I chose to implement a conservative default approach,
accepting that in some cases the user will be prompted for duplicate inputs for the same value.
To remedy that, the pipeline creator has the option to manually specify a mapping to rename some or all of
the input/output fields.
Further, because in a lot of cases there won't be any overlapping fields, the creator can specify ``auto``,
in which case *Kiara* will automatically create a mapping that tries to map autogenerated field names
to the shortest possible names for each case.
Examples:
Configuration for a pipeline module that functions as a ``nand`` logic gate (in Python):
``` python
and_step = PipelineStepConfig(module_type="and", step_id="and")
not_step = PipelineStepConfig(module_type="not", step_id="not", input_links={"a": ["and.y"]}
nand_p_conf = PipelineConfig(doc="Returns 'False' if both inputs are 'True'.",
steps=[and_step, not_step],
input_aliases={
"and__a": "a",
"and__b": "b"
},
output_aliases={
"not__y": "y"
}}
```
Or, the same thing in json:
``` json
{
"module_type_name": "nand",
"doc": "Returns 'False' if both inputs are 'True'.",
"steps": [
{
"module_type": "and",
"step_id": "and"
},
{
"module_type": "not",
"step_id": "not",
"input_links": {
"a": "and.y"
}
}
],
"input_aliases": {
"and__a": "a",
"and__b": "b"
},
"output_aliases": {
"not__y": "y"
}
}
```
"""
_kiara_model_id = "instance.module_config.pipeline"
@classmethod
def from_file(
cls,
path: str,
kiara: Union["Kiara", None] = None,
# module_map: Optional[Mapping[str, Any]] = None,
):
data = get_data_from_file(path)
pipeline_name = data.pop("pipeline_name", None)
if pipeline_name is None:
pipeline_name = os.path.basename(path)
pipeline_dir = os.path.abspath(os.path.dirname(path))
execution_context = ExecutionContext(pipeline_dir=pipeline_dir)
return cls.from_config(
pipeline_name=pipeline_name,
data=data,
kiara=kiara,
execution_context=execution_context,
)
@classmethod
def from_config(
cls,
pipeline_name: str,
data: Mapping[str, Any],
kiara: Union["Kiara", None] = None,
module_map: Union[Mapping[str, Any], None] = None,
execution_context: Union[ExecutionContext, None] = None,
auto_step_ids: bool = False,
):
if kiara is None:
from kiara.context import Kiara
kiara = Kiara.instance()
if not kiara.operation_registry.is_initialized:
kiara.operation_registry.operations # noqa
if execution_context is None:
execution_context = ExecutionContext()
config = cls._from_config(
pipeline_name=pipeline_name,
data=data,
kiara=kiara,
module_map=module_map,
execution_context=execution_context,
auto_step_ids=auto_step_ids,
)
return config
@classmethod
def _from_config(
cls,
pipeline_name: str,
data: Mapping[str, Any],
kiara: "Kiara",
module_map: Union[Mapping[str, Any], None] = None,
execution_context: Union[ExecutionContext, None] = None,
auto_step_ids: bool = False,
):
if execution_context is None:
execution_context = ExecutionContext()
repl_dict = execution_context.dict()
data = dict(data)
steps = data.pop("steps")
steps = PipelineStep.create_steps(
*steps, kiara=kiara, module_map=module_map, auto_step_ids=auto_step_ids
)
data["steps"] = steps
if not data.get("input_aliases"):
data["input_aliases"] = create_input_alias_map(steps)
if not data.get("output_aliases"):
data["output_aliases"] = create_output_alias_map(steps)
if "defaults" in data.keys():
defaults = data.pop("defaults")
replaced = replace_var_names_in_obj(defaults, repl_dict=repl_dict)
data["defaults"] = replaced
if "constants" in data.keys():
constants = data.pop("constants")
replaced = replace_var_names_in_obj(constants, repl_dict=repl_dict)
data["constants"] = replaced
if "inputs" in data.keys():
inputs = data.pop("inputs")
replaced = replace_var_names_in_obj(inputs, repl_dict=repl_dict)
data["inputs"] = replaced
result = cls(pipeline_name=pipeline_name, **data)
return result
class Config:
extra = Extra.ignore
validate_assignment = True
pipeline_name: str = Field(description="The name of this pipeline.")
steps: List[PipelineStep] = Field(
description="A list of steps/modules of this pipeline, and their connections.",
)
input_aliases: Dict[str, str] = Field(
description="A map of input aliases, with the location of the input (in the format '[step_id].[input_field]') as key, and the pipeline input field name as value.",
)
output_aliases: Dict[str, str] = Field(
description="A map of output aliases, with the location of the output (in the format '[step_id].[output_field]') as key, and the pipeline output field name as value.",
)
doc: DocumentationMetadataModel = Field(
default="-- n/a --", description="Documentation about what the pipeline does."
)
context: Dict[str, Any] = Field(
default_factory=dict, description="Metadata for this workflow."
)
_structure: Union["PipelineStructure", None] = PrivateAttr(default=None)
@validator("doc", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
@validator("steps", pre=True)
def _validate_steps(cls, v):
# if not v:
# raise ValueError(f"Invalid type for 'steps' value: {type(v)}")
steps = []
for step in v:
if not step:
raise ValueError("No step data provided.")
if isinstance(step, PipelineStep):
steps.append(step)
elif isinstance(step, Mapping):
steps.append(PipelineStep(**step))
else:
raise TypeError(step)
return steps
@property
def structure(self) -> "PipelineStructure":
if self._structure is not None:
return self._structure
from kiara.models.module.pipeline.structure import PipelineStructure
self._structure = PipelineStructure(pipeline_config=self)
return self._structure
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
table.add_row("doc", self.doc.full_doc)
table.add_row("structure", self.structure)
return table
# return create_table_from_model_object(self, exclude_fields={"steps"})
return create_table_from_model_object(self)
# def create_input_alias_map(self) -> Dict[str, str]:
#
# aliases: Dict[str, List[str]] = {}
# for step in self.steps:
# field_names = step.module.input_names
# for field_name in field_names:
# aliases.setdefault(field_name, []).append(step.step_id)
#
# result: Dict[str, str] = {}
# for field_name, step_ids in aliases.items():
# for step_id in step_ids:
# generated = generate_pipeline_endpoint_name(step_id, field_name)
# result[generated] = generated
#
# return result
#
# def create_output_alias_map(self) -> Dict[str, str]:
#
# aliases: Dict[str, List[str]] = {}
# for step in self.steps:
# field_names = step.module.input_names
# for field_name in field_names:
# aliases.setdefault(field_name, []).append(step.step_id)
#
# result: Dict[str, str] = {}
# for field_name, step_ids in aliases.items():
# for step_id in step_ids:
# generated = generate_pipeline_endpoint_name(step_id, field_name)
# result[generated] = generated
#
# return result
context: Dict[str, Any]
pydantic-field
¶Metadata for this workflow.
doc: DocumentationMetadataModel
pydantic-field
¶Documentation about what the pipeline does.
input_aliases: Dict[str, str]
pydantic-field
required
¶A map of input aliases, with the location of the input (in the format '[step_id].[input_field]') as key, and the pipeline input field name as value.
output_aliases: Dict[str, str]
pydantic-field
required
¶A map of output aliases, with the location of the output (in the format '[step_id].[output_field]') as key, and the pipeline output field name as value.
pipeline_name: str
pydantic-field
required
¶The name of this pipeline.
steps: List[kiara.models.module.pipeline.PipelineStep]
pydantic-field
required
¶A list of steps/modules of this pipeline, and their connections.
structure: PipelineStructure
property
readonly
¶
Config
¶create_renderable(self, **config)
¶Source code in kiara/modules/included_core_modules/pipeline.py
def create_renderable(self, **config: Any) -> RenderableType:
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
table.add_row("doc", self.doc.full_doc)
table.add_row("structure", self.structure)
return table
# return create_table_from_model_object(self, exclude_fields={"steps"})
return create_table_from_model_object(self)
from_config(pipeline_name, data, kiara=None, module_map=None, execution_context=None, auto_step_ids=False)
classmethod
¶Source code in kiara/modules/included_core_modules/pipeline.py
@classmethod
def from_config(
cls,
pipeline_name: str,
data: Mapping[str, Any],
kiara: Union["Kiara", None] = None,
module_map: Union[Mapping[str, Any], None] = None,
execution_context: Union[ExecutionContext, None] = None,
auto_step_ids: bool = False,
):
if kiara is None:
from kiara.context import Kiara
kiara = Kiara.instance()
if not kiara.operation_registry.is_initialized:
kiara.operation_registry.operations # noqa
if execution_context is None:
execution_context = ExecutionContext()
config = cls._from_config(
pipeline_name=pipeline_name,
data=data,
kiara=kiara,
module_map=module_map,
execution_context=execution_context,
auto_step_ids=auto_step_ids,
)
return config
from_file(path, kiara=None)
classmethod
¶Source code in kiara/modules/included_core_modules/pipeline.py
@classmethod
def from_file(
cls,
path: str,
kiara: Union["Kiara", None] = None,
# module_map: Optional[Mapping[str, Any]] = None,
):
data = get_data_from_file(path)
pipeline_name = data.pop("pipeline_name", None)
if pipeline_name is None:
pipeline_name = os.path.basename(path)
pipeline_dir = os.path.abspath(os.path.dirname(path))
execution_context = ExecutionContext(pipeline_dir=pipeline_dir)
return cls.from_config(
pipeline_name=pipeline_name,
data=data,
kiara=kiara,
execution_context=execution_context,
)
validate_doc(value)
classmethod
¶Source code in kiara/modules/included_core_modules/pipeline.py
@validator("doc", pre=True)
def validate_doc(cls, value):
return DocumentationMetadataModel.create(value)
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/pipeline.py
def create_inputs_schema(
self,
) -> ValueMapSchema:
pipeline_structure: PipelineStructure = self.config.structure
inputs_schema = pipeline_structure.pipeline_inputs_schema
return inputs_schema
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/pipeline.py
def create_outputs_schema(
self,
) -> ValueMapSchema:
pipeline_structure: PipelineStructure = self.config.structure
return pipeline_structure.pipeline_outputs_schema
is_pipeline()
classmethod
¶Check whether this module type is a pipeline, or not.
Source code in kiara/modules/included_core_modules/pipeline.py
@classmethod
def is_pipeline(cls) -> bool:
return True
process(self, inputs, outputs, job_log)
¶Source code in kiara/modules/included_core_modules/pipeline.py
def process(self, inputs: ValueMap, outputs: ValueMapWritable, job_log: JobLog):
pipeline_structure: PipelineStructure = self.config.structure
pipeline = Pipeline(structure=pipeline_structure, kiara=outputs._kiara)
assert self._job_registry is not None
controller = SinglePipelineBatchController(
pipeline=pipeline, job_registry=self._job_registry
)
pipeline.set_pipeline_inputs(inputs=inputs)
controller.process_pipeline()
# TODO: resolve values first?
outputs.set_values(**pipeline.get_current_pipeline_outputs())
pretty_print
¶
PrettyPrintAnyValueModule (PrettyPrintModule)
¶Source code in kiara/modules/included_core_modules/pretty_print.py
class PrettyPrintAnyValueModule(PrettyPrintModule):
_module_type_name = "pretty_print.any.value"
# def pretty_print__any__as__string(self, value: Value, render_config: Dict[str, Any]):
#
# data = value.data
# if isinstance(data, KiaraModel):
# return data.json(option=orjson.OPT_INDENT_2)
# else:
# return str(data)
PrettyPrintConfig (KiaraModuleConfig)
pydantic-model
¶Source code in kiara/modules/included_core_modules/pretty_print.py
class PrettyPrintConfig(KiaraModuleConfig):
source_type: str = Field(description="The value type of the source value.")
target_type: str = Field(description="The value type of the rendered value.")
@validator("source_type")
def validate_source_type(cls, value):
if value == "render_config":
raise ValueError(f"Invalid source type: {value}.")
return value
source_type: str
pydantic-field
required
¶The value type of the source value.
target_type: str
pydantic-field
required
¶The value type of the rendered value.
validate_source_type(value)
classmethod
¶Source code in kiara/modules/included_core_modules/pretty_print.py
@validator("source_type")
def validate_source_type(cls, value):
if value == "render_config":
raise ValueError(f"Invalid source type: {value}.")
return value
PrettyPrintModule (KiaraModule)
¶Source code in kiara/modules/included_core_modules/pretty_print.py
class PrettyPrintModule(KiaraModule):
_module_type_name: str = None # type: ignore
_config_cls = PrettyPrintConfig
@classmethod
def retrieve_supported_render_combinations(cls) -> Iterable[Tuple[str, str]]:
result = []
for attr in dir(cls):
if (
len(attr) <= 19
or not attr.startswith("pretty_print__")
or "__as__" not in attr
):
continue
attr = attr[14:]
end_start_type = attr.find("__as__")
source_type = attr[0:end_start_type]
target_type = attr[end_start_type + 6 :] # noqa
result.append((source_type, target_type))
return result
# def create_persistence_config_schema(self) -> Optional[Mapping[str, Mapping[str, Any]]]:
# return None
def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
return DEFAULT_IDEMPOTENT_INTERNAL_MODULE_CHARACTERISTICS
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
source_type = self.get_config_value("source_type")
assert source_type not in ["target", "base_name"]
schema = {
"value": {"type": source_type, "doc": "The value to render."},
"render_config": {
"type": "any",
"doc": "Value type dependent render configuration.",
"optional": True,
},
}
return schema
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return {
"rendered_value": {
"type": self.get_config_value("target_type"),
"doc": "The rendered value.",
}
}
def process(self, inputs: ValueMap, outputs: ValueMap):
source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
value = inputs.get_value_obj("value")
render_config = inputs.get_value_data("render_config")
func_name = f"pretty_print__{source_type}__as__{target_type}"
func = getattr(self, func_name)
# TODO: check function signature is valid
if render_config is None:
render_config = {}
result = func(value=value, render_config=render_config)
outputs.set_value("rendered_value", result)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶Source code in kiara/modules/included_core_modules/pretty_print.py
class PrettyPrintConfig(KiaraModuleConfig):
source_type: str = Field(description="The value type of the source value.")
target_type: str = Field(description="The value type of the rendered value.")
@validator("source_type")
def validate_source_type(cls, value):
if value == "render_config":
raise ValueError(f"Invalid source type: {value}.")
return value
source_type: str
pydantic-field
required
¶The value type of the source value.
target_type: str
pydantic-field
required
¶The value type of the rendered value.
validate_source_type(value)
classmethod
¶Source code in kiara/modules/included_core_modules/pretty_print.py
@validator("source_type")
def validate_source_type(cls, value):
if value == "render_config":
raise ValueError(f"Invalid source type: {value}.")
return value
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/pretty_print.py
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
source_type = self.get_config_value("source_type")
assert source_type not in ["target", "base_name"]
schema = {
"value": {"type": source_type, "doc": "The value to render."},
"render_config": {
"type": "any",
"doc": "Value type dependent render configuration.",
"optional": True,
},
}
return schema
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/pretty_print.py
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return {
"rendered_value": {
"type": self.get_config_value("target_type"),
"doc": "The rendered value.",
}
}
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/pretty_print.py
def process(self, inputs: ValueMap, outputs: ValueMap):
source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
value = inputs.get_value_obj("value")
render_config = inputs.get_value_data("render_config")
func_name = f"pretty_print__{source_type}__as__{target_type}"
func = getattr(self, func_name)
# TODO: check function signature is valid
if render_config is None:
render_config = {}
result = func(value=value, render_config=render_config)
outputs.set_value("rendered_value", result)
retrieve_supported_render_combinations()
classmethod
¶Source code in kiara/modules/included_core_modules/pretty_print.py
@classmethod
def retrieve_supported_render_combinations(cls) -> Iterable[Tuple[str, str]]:
result = []
for attr in dir(cls):
if (
len(attr) <= 19
or not attr.startswith("pretty_print__")
or "__as__" not in attr
):
continue
attr = attr[14:]
end_start_type = attr.find("__as__")
source_type = attr[0:end_start_type]
target_type = attr[end_start_type + 6 :] # noqa
result.append((source_type, target_type))
return result
ValueTypePrettyPrintModule (KiaraModule)
¶Source code in kiara/modules/included_core_modules/pretty_print.py
class ValueTypePrettyPrintModule(KiaraModule):
_module_type_name = "pretty_print.value"
_config_cls = PrettyPrintConfig
def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
return DEFAULT_IDEMPOTENT_INTERNAL_MODULE_CHARACTERISTICS
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
source_type = self.get_config_value("source_type")
assert source_type not in ["target", "base_name"]
schema = {
"value": {
"type": source_type,
"doc": "The value to render.",
"optional": True,
},
"render_config": {
"type": "any",
"doc": "Value type dependent render configuration.",
"optional": True,
},
}
return schema
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return {
"rendered_value": {
"type": self.get_config_value("target_type"),
"doc": "The rendered value.",
}
}
def process(self, inputs: ValueMap, outputs: ValueMap):
# source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
source_value = inputs.get_value_obj("value")
render_config = inputs.get_value_obj("render_config")
if not source_value.is_set:
outputs.set_value("rendered_value", "-- none/not set --")
return
data_type_cls = source_value.data_type_info.data_type_class.get_class()
data_type = data_type_cls(**source_value.value_schema.type_config)
func_name = f"pretty_print_as__{target_type}"
func = getattr(data_type, func_name)
render_config_dict = render_config.data
if render_config_dict is None:
render_config_dict = {}
result = func(value=source_value, render_config=render_config_dict)
# TODO: check we have the correct type?
outputs.set_value("rendered_value", result)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶Source code in kiara/modules/included_core_modules/pretty_print.py
class PrettyPrintConfig(KiaraModuleConfig):
source_type: str = Field(description="The value type of the source value.")
target_type: str = Field(description="The value type of the rendered value.")
@validator("source_type")
def validate_source_type(cls, value):
if value == "render_config":
raise ValueError(f"Invalid source type: {value}.")
return value
source_type: str
pydantic-field
required
¶The value type of the source value.
target_type: str
pydantic-field
required
¶The value type of the rendered value.
validate_source_type(value)
classmethod
¶Source code in kiara/modules/included_core_modules/pretty_print.py
@validator("source_type")
def validate_source_type(cls, value):
if value == "render_config":
raise ValueError(f"Invalid source type: {value}.")
return value
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/pretty_print.py
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
source_type = self.get_config_value("source_type")
assert source_type not in ["target", "base_name"]
schema = {
"value": {
"type": source_type,
"doc": "The value to render.",
"optional": True,
},
"render_config": {
"type": "any",
"doc": "Value type dependent render configuration.",
"optional": True,
},
}
return schema
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/pretty_print.py
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return {
"rendered_value": {
"type": self.get_config_value("target_type"),
"doc": "The rendered value.",
}
}
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/pretty_print.py
def process(self, inputs: ValueMap, outputs: ValueMap):
# source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
source_value = inputs.get_value_obj("value")
render_config = inputs.get_value_obj("render_config")
if not source_value.is_set:
outputs.set_value("rendered_value", "-- none/not set --")
return
data_type_cls = source_value.data_type_info.data_type_class.get_class()
data_type = data_type_cls(**source_value.value_schema.type_config)
func_name = f"pretty_print_as__{target_type}"
func = getattr(data_type, func_name)
render_config_dict = render_config.data
if render_config_dict is None:
render_config_dict = {}
result = func(value=source_value, render_config=render_config_dict)
# TODO: check we have the correct type?
outputs.set_value("rendered_value", result)
render_value
¶
RenderValueModule (KiaraModule)
¶Source code in kiara/modules/included_core_modules/render_value.py
class RenderValueModule(KiaraModule):
@classmethod
def retrieve_supported_render_combinations(cls) -> Iterable[Tuple[str, str]]:
result = []
for attr in dir(cls):
if (
len(attr) <= 16
or not attr.startswith("render__")
or "__as__" not in attr
):
continue
attr = attr[8:]
end_start_type = attr.find("__as__")
source_type = attr[0:end_start_type]
target_type = attr[end_start_type + 6 :] # noqa
result.append((source_type, target_type))
return result
_config_cls = RenderValueModuleConfig
_module_type_name: str = None # type: ignore
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
# instruction = self.get_config_value("render_scene_type")
# model_registry = ModelRegistry.instance()
# instr_model_cls: Type[RenderScene] = model_registry.get_model_cls(instruction, required_subclass=RenderScene) # type: ignore
# data_type_name = instr_model_cls.retrieve_source_type()
# assert data_type_name
source_type = self.get_config_value("source_type")
optional = source_type == "none"
inputs = {
"value": {
"type": source_type,
"doc": f"A value of type '{source_type}'",
"optional": optional,
},
"render_config": {
"type": "dict",
"doc": "Instructions/config on how (or what) to render the provided value.",
"default": {},
},
}
return inputs
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
outputs = {
"render_value_result": {
"type": "render_value_result",
"doc": "The rendered value, incl. some metadata.",
},
}
return outputs
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
value: Value = inputs.get_value_obj("value")
render_scene: DictModel = inputs.get_value_data("render_config")
func_name = f"render__{source_type}__as__{target_type}"
func = getattr(self, func_name)
result = func(value=value, render_config=render_scene.dict_data)
if isinstance(result, RenderValueResult):
render_scene_result: RenderValueResult = result
else:
render_scene_result = RenderValueResult(
value_id=value.value_id,
render_config=render_scene,
render_manifest=self.manifest.manifest_hash,
rendered=result,
related_scenes={},
)
render_scene_result.manifest_lookup[self.manifest.manifest_hash] = self.manifest
outputs.set_value("render_value_result", render_scene_result)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶Source code in kiara/modules/included_core_modules/render_value.py
class RenderValueModuleConfig(KiaraModuleConfig):
# render_scene_type: str = Field(
# description="The id of the model that describes (and handles) the actual rendering."
# )
source_type: str = Field(description="The (kiara) data type to be rendered.")
target_type: str = Field(
description="The (kiara) data type of the rendered result."
)
# @validator("render_scene_type")
# def validate_render_scene(cls, value: Any):
#
# registry = ModelRegistry.instance()
#
# if value not in registry.all_models.item_infos.keys():
# raise ValueError(
# f"Invalid model type '{value}'. Value model ids: {', '.join(registry.all_models.item_infos.keys())}."
# )
#
# return value
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/render_value.py
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
# instruction = self.get_config_value("render_scene_type")
# model_registry = ModelRegistry.instance()
# instr_model_cls: Type[RenderScene] = model_registry.get_model_cls(instruction, required_subclass=RenderScene) # type: ignore
# data_type_name = instr_model_cls.retrieve_source_type()
# assert data_type_name
source_type = self.get_config_value("source_type")
optional = source_type == "none"
inputs = {
"value": {
"type": source_type,
"doc": f"A value of type '{source_type}'",
"optional": optional,
},
"render_config": {
"type": "dict",
"doc": "Instructions/config on how (or what) to render the provided value.",
"default": {},
},
}
return inputs
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/render_value.py
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
outputs = {
"render_value_result": {
"type": "render_value_result",
"doc": "The rendered value, incl. some metadata.",
},
}
return outputs
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/render_value.py
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
value: Value = inputs.get_value_obj("value")
render_scene: DictModel = inputs.get_value_data("render_config")
func_name = f"render__{source_type}__as__{target_type}"
func = getattr(self, func_name)
result = func(value=value, render_config=render_scene.dict_data)
if isinstance(result, RenderValueResult):
render_scene_result: RenderValueResult = result
else:
render_scene_result = RenderValueResult(
value_id=value.value_id,
render_config=render_scene,
render_manifest=self.manifest.manifest_hash,
rendered=result,
related_scenes={},
)
render_scene_result.manifest_lookup[self.manifest.manifest_hash] = self.manifest
outputs.set_value("render_value_result", render_scene_result)
retrieve_supported_render_combinations()
classmethod
¶Source code in kiara/modules/included_core_modules/render_value.py
@classmethod
def retrieve_supported_render_combinations(cls) -> Iterable[Tuple[str, str]]:
result = []
for attr in dir(cls):
if (
len(attr) <= 16
or not attr.startswith("render__")
or "__as__" not in attr
):
continue
attr = attr[8:]
end_start_type = attr.find("__as__")
source_type = attr[0:end_start_type]
target_type = attr[end_start_type + 6 :] # noqa
result.append((source_type, target_type))
return result
RenderValueModuleConfig (KiaraModuleConfig)
pydantic-model
¶Source code in kiara/modules/included_core_modules/render_value.py
class RenderValueModuleConfig(KiaraModuleConfig):
# render_scene_type: str = Field(
# description="The id of the model that describes (and handles) the actual rendering."
# )
source_type: str = Field(description="The (kiara) data type to be rendered.")
target_type: str = Field(
description="The (kiara) data type of the rendered result."
)
# @validator("render_scene_type")
# def validate_render_scene(cls, value: Any):
#
# registry = ModelRegistry.instance()
#
# if value not in registry.all_models.item_infos.keys():
# raise ValueError(
# f"Invalid model type '{value}'. Value model ids: {', '.join(registry.all_models.item_infos.keys())}."
# )
#
# return value
ValueTypeRenderModule (KiaraModule)
¶A module that uses render methods attached to DataType classes.
Source code in kiara/modules/included_core_modules/render_value.py
class ValueTypeRenderModule(KiaraModule):
"""A module that uses render methods attached to DataType classes."""
_module_type_name = "render.value"
_config_cls = RenderValueModuleConfig
def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
return DEFAULT_IDEMPOTENT_INTERNAL_MODULE_CHARACTERISTICS
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
source_type = self.get_config_value("source_type")
assert source_type not in ["target", "base_name"]
schema = {
"value": {
"type": source_type,
"doc": "The value to render.",
"optional": False,
},
"render_config": {
"type": "dict",
"doc": "Instructions/config on how (or what) to render the provided value.",
"default": {},
},
}
return schema
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
outputs = {
"render_value_result": {
"type": "render_value_result",
"doc": "The rendered value, incl. some metadata.",
},
}
return outputs
def process(self, inputs: ValueMap, outputs: ValueMap):
source_value = inputs.get_value_obj("value")
if not source_value.is_set:
raise KiaraProcessingException(
f"Can't render value '{source_value.value_id}': value not set."
)
# source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
render_scene: DictModel = inputs.get_value_data("render_config")
data_type_cls = source_value.data_type_info.data_type_class.get_class()
data_type = data_type_cls(**source_value.value_schema.type_config)
func_name = f"render_as__{target_type}"
func = getattr(data_type, func_name)
result = func(
value=source_value,
render_config=render_scene.dict_data,
manifest=self.manifest,
)
if isinstance(result, RenderValueResult):
render_scene_result = result
else:
render_scene_result = RenderValueResult(
value_id=source_value.value_id,
render_config=render_scene,
render_manifest=self.manifest.manifest_hash,
rendered=result,
related_scenes={},
manifest_lookup={self.manifest.manifest_hash: self.manifest},
)
outputs.set_value("render_value_result", render_scene_result)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶Source code in kiara/modules/included_core_modules/render_value.py
class RenderValueModuleConfig(KiaraModuleConfig):
# render_scene_type: str = Field(
# description="The id of the model that describes (and handles) the actual rendering."
# )
source_type: str = Field(description="The (kiara) data type to be rendered.")
target_type: str = Field(
description="The (kiara) data type of the rendered result."
)
# @validator("render_scene_type")
# def validate_render_scene(cls, value: Any):
#
# registry = ModelRegistry.instance()
#
# if value not in registry.all_models.item_infos.keys():
# raise ValueError(
# f"Invalid model type '{value}'. Value model ids: {', '.join(registry.all_models.item_infos.keys())}."
# )
#
# return value
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/render_value.py
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
source_type = self.get_config_value("source_type")
assert source_type not in ["target", "base_name"]
schema = {
"value": {
"type": source_type,
"doc": "The value to render.",
"optional": False,
},
"render_config": {
"type": "dict",
"doc": "Instructions/config on how (or what) to render the provided value.",
"default": {},
},
}
return schema
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/render_value.py
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
outputs = {
"render_value_result": {
"type": "render_value_result",
"doc": "The rendered value, incl. some metadata.",
},
}
return outputs
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/render_value.py
def process(self, inputs: ValueMap, outputs: ValueMap):
source_value = inputs.get_value_obj("value")
if not source_value.is_set:
raise KiaraProcessingException(
f"Can't render value '{source_value.value_id}': value not set."
)
# source_type = self.get_config_value("source_type")
target_type = self.get_config_value("target_type")
render_scene: DictModel = inputs.get_value_data("render_config")
data_type_cls = source_value.data_type_info.data_type_class.get_class()
data_type = data_type_cls(**source_value.value_schema.type_config)
func_name = f"render_as__{target_type}"
func = getattr(data_type, func_name)
result = func(
value=source_value,
render_config=render_scene.dict_data,
manifest=self.manifest,
)
if isinstance(result, RenderValueResult):
render_scene_result = result
else:
render_scene_result = RenderValueResult(
value_id=source_value.value_id,
render_config=render_scene,
render_manifest=self.manifest.manifest_hash,
rendered=result,
related_scenes={},
manifest_lookup={self.manifest.manifest_hash: self.manifest},
)
outputs.set_value("render_value_result", render_scene_result)
serialization
¶
DeserializeFromJsonModule (KiaraModule)
¶Source code in kiara/modules/included_core_modules/serialization.py
class DeserializeFromJsonModule(KiaraModule):
_module_type_name: str = "deserialize.from_json"
_config_cls = DeserializeJsonConfig
def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
return DEFAULT_IDEMPOTENT_INTERNAL_MODULE_CHARACTERISTICS
def create_inputs_schema(
self,
) -> ValueMapSchema:
return {
"value": {
"type": "any",
"doc": "The value object to deserialize the data for.",
}
}
def create_outputs_schema(
self,
) -> ValueMapSchema:
return {
"python_object": {
"type": "python_object",
"doc": "The deserialized python object.",
}
}
def process(self, inputs: ValueMap, outputs: ValueMap):
value: Value = inputs.get_value_obj("value")
serialized: SerializedData = value.serialized_data
chunks = serialized.get_serialized_data(self.get_config_value("result_path"))
assert chunks.get_number_of_chunks() == 1
_data = list(chunks.get_chunks(as_files=False))
assert len(_data) == 1
_chunk: bytes = _data[0] # type: ignore
deserialized = orjson.loads(_chunk)
outputs.set_value("python_object", deserialized)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶Source code in kiara/modules/included_core_modules/serialization.py
class DeserializeJsonConfig(KiaraModuleConfig):
result_path: Union[str, None] = Field(
description="The path of the result dictionary to return.", default="data"
)
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/serialization.py
def create_inputs_schema(
self,
) -> ValueMapSchema:
return {
"value": {
"type": "any",
"doc": "The value object to deserialize the data for.",
}
}
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/serialization.py
def create_outputs_schema(
self,
) -> ValueMapSchema:
return {
"python_object": {
"type": "python_object",
"doc": "The deserialized python object.",
}
}
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/serialization.py
def process(self, inputs: ValueMap, outputs: ValueMap):
value: Value = inputs.get_value_obj("value")
serialized: SerializedData = value.serialized_data
chunks = serialized.get_serialized_data(self.get_config_value("result_path"))
assert chunks.get_number_of_chunks() == 1
_data = list(chunks.get_chunks(as_files=False))
assert len(_data) == 1
_chunk: bytes = _data[0] # type: ignore
deserialized = orjson.loads(_chunk)
outputs.set_value("python_object", deserialized)
DeserializeJsonConfig (KiaraModuleConfig)
pydantic-model
¶Source code in kiara/modules/included_core_modules/serialization.py
class DeserializeJsonConfig(KiaraModuleConfig):
result_path: Union[str, None] = Field(
description="The path of the result dictionary to return.", default="data"
)
DeserializeValueModule (KiaraModule)
¶Source code in kiara/modules/included_core_modules/serialization.py
class DeserializeValueModule(KiaraModule):
_config_cls = SerializeConfig
@classmethod
@abc.abstractmethod
def retrieve_serialized_value_type(cls) -> str:
raise NotImplementedError()
@classmethod
@abc.abstractmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
raise NotImplementedError()
@classmethod
@abc.abstractmethod
def retrieve_supported_serialization_profile(cls) -> str:
raise NotImplementedError()
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
value_type = self.get_config_value("value_type")
return {
value_type: {
"type": value_type,
"doc": "The value object.",
},
"deserialization_config": {
"type": "any",
"doc": "Serialization-format specific configuration.",
"optional": True,
},
}
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return {
"python_object": {
"type": "python_object",
"doc": "The deserialized python object instance.",
},
}
def _retrieve_module_characteristics(self) -> ModuleCharacteristics:
return DEFAULT_IDEMPOTENT_INTERNAL_MODULE_CHARACTERISTICS
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
value_type = self.get_config_value("value_type")
serialized_value = inputs.get_value_obj(value_type)
config = inputs.get_value_obj("deserialization_config")
target_profile = self.get_config_value("target_profile")
func_name = f"to__{target_profile}"
func = getattr(self, func_name)
if config.is_set:
_config = config.data
else:
_config = {}
result: Any = func(data=serialized_value.serialized_data, **_config)
outputs.set_value("python_object", result)
_config_cls (KiaraModuleConfig)
private
pydantic-model
¶Source code in kiara/modules/included_core_modules/serialization.py
class SerializeConfig(KiaraModuleConfig):
value_type: str = Field(
description="The value type of the actual (unserialized) value."
)
target_profile: str = Field(
description="The profile name of the de-serialization result data."
)
serialization_profile: str = Field(
description="The name of the serialization profile used to serialize the source value."
)
@validator("value_type")
def validate_source_type(cls, value):
if value == "serialization_config":
raise ValueError(f"Invalid source type: {value}.")
return value
serialization_profile: str
pydantic-field
required
¶The name of the serialization profile used to serialize the source value.
target_profile: str
pydantic-field
required
¶The profile name of the de-serialization result data.
value_type: str
pydantic-field
required
¶The value type of the actual (unserialized) value.
validate_source_type(value)
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@validator("value_type")
def validate_source_type(cls, value):
if value == "serialization_config":
raise ValueError(f"Invalid source type: {value}.")
return value
create_inputs_schema(self)
¶Return the schema for this types' inputs.
Source code in kiara/modules/included_core_modules/serialization.py
def create_inputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
value_type = self.get_config_value("value_type")
return {
value_type: {
"type": value_type,
"doc": "The value object.",
},
"deserialization_config": {
"type": "any",
"doc": "Serialization-format specific configuration.",
"optional": True,
},
}
create_outputs_schema(self)
¶Return the schema for this types' outputs.
Source code in kiara/modules/included_core_modules/serialization.py
def create_outputs_schema(
self,
) -> Mapping[str, Union[ValueSchema, Mapping[str, Any]]]:
return {
"python_object": {
"type": "python_object",
"doc": "The deserialized python object instance.",
},
}
process(self, inputs, outputs)
¶Source code in kiara/modules/included_core_modules/serialization.py
def process(self, inputs: ValueMap, outputs: ValueMap) -> None:
value_type = self.get_config_value("value_type")
serialized_value = inputs.get_value_obj(value_type)
config = inputs.get_value_obj("deserialization_config")
target_profile = self.get_config_value("target_profile")
func_name = f"to__{target_profile}"
func = getattr(self, func_name)
if config.is_set:
_config = config.data
else:
_config = {}
result: Any = func(data=serialized_value.serialized_data, **_config)
outputs.set_value("python_object", result)
retrieve_serialized_value_type()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
@abc.abstractmethod
def retrieve_serialized_value_type(cls) -> str:
raise NotImplementedError()
retrieve_supported_serialization_profile()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
@abc.abstractmethod
def retrieve_supported_serialization_profile(cls) -> str:
raise NotImplementedError()
retrieve_supported_target_profiles()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
@abc.abstractmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
raise NotImplementedError()
LoadBytesModule (DeserializeValueModule)
¶Source code in kiara/modules/included_core_modules/serialization.py
class LoadBytesModule(DeserializeValueModule):
_module_type_name = "load.bytes"
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": bytes}
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "raw"
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "bytes"
def to__python_object(self, data: SerializedData, **config: Any) -> bytes:
chunks = data.get_serialized_data("bytes")
assert chunks.get_number_of_chunks() == 1
_chunks = list(chunks.get_chunks(as_files=False))
assert len(_chunks) == 1
_chunk: bytes = _chunks[0] # type: ignore
return _chunk
retrieve_serialized_value_type()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "bytes"
retrieve_supported_serialization_profile()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "raw"
retrieve_supported_target_profiles()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": bytes}
to__python_object(self, data, **config)
¶Source code in kiara/modules/included_core_modules/serialization.py
def to__python_object(self, data: SerializedData, **config: Any) -> bytes:
chunks = data.get_serialized_data("bytes")
assert chunks.get_number_of_chunks() == 1
_chunks = list(chunks.get_chunks(as_files=False))
assert len(_chunks) == 1
_chunk: bytes = _chunks[0] # type: ignore
return _chunk
LoadInternalModel (DeserializeValueModule)
¶Source code in kiara/modules/included_core_modules/serialization.py
class LoadInternalModel(DeserializeValueModule):
_module_type_name = "load.internal_model"
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": KiaraModel}
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "json"
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "internal_model"
def to__python_object(self, data: SerializedData, **config: Any) -> KiaraModel:
chunks = data.get_serialized_data("data")
assert chunks.get_number_of_chunks() == 1
_chunks = list(chunks.get_chunks(as_files=False))
assert len(_chunks) == 1
bytes_string: bytes = _chunks[0] # type: ignore
model_data = orjson.loads(bytes_string)
model_id: str = data.data_type_config["kiara_model_id"]
model_registry = ModelRegistry.instance()
m_cls = model_registry.get_model_cls(kiara_model_id=model_id)
obj = m_cls(**model_data)
return obj
retrieve_serialized_value_type()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "internal_model"
retrieve_supported_serialization_profile()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "json"
retrieve_supported_target_profiles()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": KiaraModel}
to__python_object(self, data, **config)
¶Source code in kiara/modules/included_core_modules/serialization.py
def to__python_object(self, data: SerializedData, **config: Any) -> KiaraModel:
chunks = data.get_serialized_data("data")
assert chunks.get_number_of_chunks() == 1
_chunks = list(chunks.get_chunks(as_files=False))
assert len(_chunks) == 1
bytes_string: bytes = _chunks[0] # type: ignore
model_data = orjson.loads(bytes_string)
model_id: str = data.data_type_config["kiara_model_id"]
model_registry = ModelRegistry.instance()
m_cls = model_registry.get_model_cls(kiara_model_id=model_id)
obj = m_cls(**model_data)
return obj
LoadStringModule (DeserializeValueModule)
¶Source code in kiara/modules/included_core_modules/serialization.py
class LoadStringModule(DeserializeValueModule):
_module_type_name = "load.string"
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": str}
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "raw"
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "string"
def to__python_object(self, data: SerializedData, **config: Any) -> str:
chunks = data.get_serialized_data("string")
assert chunks.get_number_of_chunks() == 1
_chunks = list(chunks.get_chunks(as_files=False))
assert len(_chunks) == 1
bytes_string: bytes = _chunks[0] # type: ignore
return bytes_string.decode("utf-8")
retrieve_serialized_value_type()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "string"
retrieve_supported_serialization_profile()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "raw"
retrieve_supported_target_profiles()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": str}
to__python_object(self, data, **config)
¶Source code in kiara/modules/included_core_modules/serialization.py
def to__python_object(self, data: SerializedData, **config: Any) -> str:
chunks = data.get_serialized_data("string")
assert chunks.get_number_of_chunks() == 1
_chunks = list(chunks.get_chunks(as_files=False))
assert len(_chunks) == 1
bytes_string: bytes = _chunks[0] # type: ignore
return bytes_string.decode("utf-8")
SerializeConfig (KiaraModuleConfig)
pydantic-model
¶Source code in kiara/modules/included_core_modules/serialization.py
class SerializeConfig(KiaraModuleConfig):
value_type: str = Field(
description="The value type of the actual (unserialized) value."
)
target_profile: str = Field(
description="The profile name of the de-serialization result data."
)
serialization_profile: str = Field(
description="The name of the serialization profile used to serialize the source value."
)
@validator("value_type")
def validate_source_type(cls, value):
if value == "serialization_config":
raise ValueError(f"Invalid source type: {value}.")
return value
serialization_profile: str
pydantic-field
required
¶The name of the serialization profile used to serialize the source value.
target_profile: str
pydantic-field
required
¶The profile name of the de-serialization result data.
value_type: str
pydantic-field
required
¶The value type of the actual (unserialized) value.
validate_source_type(value)
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@validator("value_type")
def validate_source_type(cls, value):
if value == "serialization_config":
raise ValueError(f"Invalid source type: {value}.")
return value
UnpickleModule (DeserializeValueModule)
¶Source code in kiara/modules/included_core_modules/serialization.py
class UnpickleModule(DeserializeValueModule):
_module_type_name = "unpickle.value"
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": object}
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "pickle"
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "any"
def to__python_object(self, data: SerializedData, **config: Any):
try:
import pickle5 as pickle
except Exception:
import pickle # type: ignore
assert "python_object" in data.get_keys()
python_object_data = data.get_serialized_data("python_object")
assert python_object_data.get_number_of_chunks() == 1
_bytes = list(python_object_data.get_chunks(as_files=False))[0]
data = pickle.loads(_bytes)
return data
retrieve_serialized_value_type()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_serialized_value_type(cls) -> str:
return "any"
retrieve_supported_serialization_profile()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_supported_serialization_profile(cls) -> str:
return "pickle"
retrieve_supported_target_profiles()
classmethod
¶Source code in kiara/modules/included_core_modules/serialization.py
@classmethod
def retrieve_supported_target_profiles(cls) -> Mapping[str, Type]:
return {"python_object": object}
to__python_object(self, data, **config)
¶Source code in kiara/modules/included_core_modules/serialization.py
def to__python_object(self, data: SerializedData, **config: Any):
try:
import pickle5 as pickle
except Exception:
import pickle # type: ignore
assert "python_object" in data.get_keys()
python_object_data = data.get_serialized_data("python_object")
assert python_object_data.get_number_of_chunks() == 1
_bytes = list(python_object_data.get_chunks(as_files=False))[0]
data = pickle.loads(_bytes)
return data
operations
special
¶
OPERATION_TYPE_DETAILS
¶
Classes¶
OperationType (ABC, Generic)
¶
Source code in kiara/operations/__init__.py
class OperationType(abc.ABC, Generic[OPERATION_TYPE_DETAILS]):
def __init__(self, kiara: "Kiara", op_type_name: str):
self._kiara: Kiara = kiara
self._op_type_name: str = op_type_name
@property
def operations(self) -> Mapping[str, Operation]:
return {
op_id: self._kiara.operation_registry.get_operation(op_id)
for op_id in self._kiara.operation_registry.operations_by_type[
self._op_type_name
]
}
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
return []
@abc.abstractmethod
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[OPERATION_TYPE_DETAILS, None]:
"""Check whether the provided module is a valid operation for this type."""
def retrieve_operation_details(
self, operation: Union[Operation, str]
) -> OPERATION_TYPE_DETAILS:
"""Retrieve operation details for provided operation.
This is really just a utility method, to make the type checker happy.
"""
if isinstance(operation, str):
operation = self.operations[operation]
return operation.operation_details # type: ignore
operations: Mapping[str, kiara.models.module.operation.Operation]
property
readonly
¶Methods¶
check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/__init__.py
@abc.abstractmethod
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[OPERATION_TYPE_DETAILS, None]:
"""Check whether the provided module is a valid operation for this type."""
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/__init__.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
return []
retrieve_operation_details(self, operation)
¶Retrieve operation details for provided operation.
This is really just a utility method, to make the type checker happy.
Source code in kiara/operations/__init__.py
def retrieve_operation_details(
self, operation: Union[Operation, str]
) -> OPERATION_TYPE_DETAILS:
"""Retrieve operation details for provided operation.
This is really just a utility method, to make the type checker happy.
"""
if isinstance(operation, str):
operation = self.operations[operation]
return operation.operation_details # type: ignore
Modules¶
included_core_operations
special
¶
logger
¶Classes¶
CustomModuleOperationDetails (OperationDetails)
pydantic-model
¶Source code in kiara/operations/included_core_operations/__init__.py
class CustomModuleOperationDetails(OperationDetails):
@classmethod
def create_from_module(cls, module: KiaraModule):
return CustomModuleOperationDetails(
operation_id=module.module_type_name,
module_inputs_schema=module.inputs_schema,
module_outputs_schema=module.outputs_schema,
)
module_inputs_schema: Mapping[str, ValueSchema] = Field(
description="The input schemas of the module."
)
module_outputs_schema: Mapping[str, ValueSchema] = Field(
description="The output schemas of the module."
)
_op_schema: OperationSchema = PrivateAttr(default=None)
def get_operation_schema(self) -> OperationSchema:
if self._op_schema is not None:
return self._op_schema
self._op_schema = OperationSchema(
alias=self.operation_id,
inputs_schema=self.module_inputs_schema,
outputs_schema=self.module_outputs_schema,
)
return self._op_schema
# def create_module_inputs(self, inputs: Mapping[str, Any]) -> Mapping[str, Any]:
# return inputs
#
# def create_operation_outputs(self, outputs: ValueMap) -> Mapping[str, Value]:
# return outputs
module_inputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The input schemas of the module.
module_outputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The output schemas of the module.
create_from_module(module)
classmethod
¶Source code in kiara/operations/included_core_operations/__init__.py
@classmethod
def create_from_module(cls, module: KiaraModule):
return CustomModuleOperationDetails(
operation_id=module.module_type_name,
module_inputs_schema=module.inputs_schema,
module_outputs_schema=module.outputs_schema,
)
get_operation_schema(self)
¶Source code in kiara/operations/included_core_operations/__init__.py
def get_operation_schema(self) -> OperationSchema:
if self._op_schema is not None:
return self._op_schema
self._op_schema = OperationSchema(
alias=self.operation_id,
inputs_schema=self.module_inputs_schema,
outputs_schema=self.module_outputs_schema,
)
return self._op_schema
CustomModuleOperationType (OperationType)
¶Source code in kiara/operations/included_core_operations/__init__.py
class CustomModuleOperationType(OperationType[CustomModuleOperationDetails]):
_operation_type_name = "custom_module"
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = []
for name, module_cls in self._kiara.module_type_classes.items():
mod_conf = module_cls._config_cls
if mod_conf.requires_config():
logger.debug(
"ignore.custom_operation",
module_type=name,
reason="config required",
)
continue
doc = DocumentationMetadataModel.from_class_doc(module_cls)
oc = ManifestOperationConfig(module_type=name, doc=doc)
result.append(oc)
return result
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[CustomModuleOperationDetails, None]:
mod_conf = module.__class__._config_cls
if not mod_conf.requires_config():
is_internal = module.characteristics.is_internal
# inputs_map = {k: k for k in module.inputs_schema.keys()}
# outputs_map = {k: k for k in module.outputs_schema.keys()}
op_details = CustomModuleOperationDetails.create_operation_details(
operation_id=module.module_type_name,
module_inputs_schema=module.inputs_schema,
module_outputs_schema=module.outputs_schema,
is_internal_operation=is_internal,
)
return op_details
else:
return None
check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/included_core_operations/__init__.py
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[CustomModuleOperationDetails, None]:
mod_conf = module.__class__._config_cls
if not mod_conf.requires_config():
is_internal = module.characteristics.is_internal
# inputs_map = {k: k for k in module.inputs_schema.keys()}
# outputs_map = {k: k for k in module.outputs_schema.keys()}
op_details = CustomModuleOperationDetails.create_operation_details(
operation_id=module.module_type_name,
module_inputs_schema=module.inputs_schema,
module_outputs_schema=module.outputs_schema,
is_internal_operation=is_internal,
)
return op_details
else:
return None
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/included_core_operations/__init__.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = []
for name, module_cls in self._kiara.module_type_classes.items():
mod_conf = module_cls._config_cls
if mod_conf.requires_config():
logger.debug(
"ignore.custom_operation",
module_type=name,
reason="config required",
)
continue
doc = DocumentationMetadataModel.from_class_doc(module_cls)
oc = ManifestOperationConfig(module_type=name, doc=doc)
result.append(oc)
return result
Modules¶
create_from
¶logger
¶
CreateFromOperationType (OperationType)
¶Source code in kiara/operations/included_core_operations/create_from.py
class CreateFromOperationType(OperationType[CreateValueFromDetails]):
_operation_type_name = "create_from"
def _calculate_op_id(self, source_type: str, target_type: str):
if source_type == "any":
operation_id = f"create.{target_type}"
else:
operation_id = f"create.{target_type}.from.{source_type}"
return operation_id
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = {}
for name, module_cls in self._kiara.module_type_classes.items():
if not hasattr(module_cls, "retrieve_supported_create_combinations"):
continue
try:
supported_combinations = module_cls.retrieve_supported_create_combinations() # type: ignore
for sup_comb in supported_combinations:
source_type = sup_comb["source_type"]
target_type = sup_comb["target_type"]
func = sup_comb["func"]
if source_type not in self._kiara.data_type_names:
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Source type '{source_type}' not registered.",
)
continue
if target_type not in self._kiara.data_type_names:
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Target type '{target_type}' not registered.",
)
continue
if not hasattr(module_cls, func):
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Specified create function '{func}' not available.",
)
continue
mc = {"source_type": source_type, "target_type": target_type}
# TODO: check whether module config actually supports those, for now, only 'CreateFromModule' subtypes are supported
_func = getattr(module_cls, func)
doc = DocumentationMetadataModel.from_function(_func)
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
op_id = self._calculate_op_id(
source_type=source_type, target_type=target_type
)
result[op_id] = oc
except Exception as e:
log_exception(e)
logger.debug(
"ignore.create_operation_instance", module_type=name, reason=e
)
continue
return result.values()
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[CreateValueFromDetails, None]:
if not isinstance(module, CreateFromModule):
return None
source_type = None
for field_name, schema in module.inputs_schema.items():
if field_name == schema.type:
if source_type is not None:
logger.debug(
"ignore.operation",
operation_type="create_from",
reason=f"more than one possible target type field: {field_name}",
)
return None
source_type = field_name
if source_type is None:
return None
target_type = None
for field_name, schema in module.outputs_schema.items():
if field_name == schema.type:
if target_type is not None:
logger.debug(
"ignore.operation",
operation_type="create_from",
reason=f"more than one possible target type field: {field_name}",
)
return None
target_type = field_name
if target_type is None:
return None
op_id = self._calculate_op_id(source_type=source_type, target_type=target_type)
if (
"any" in self._kiara.type_registry.get_type_lineage(target_type)
and target_type != "any"
):
is_internal = False
else:
is_internal = True
optional = {}
for field, schema in module.inputs_schema.items():
if field == source_type:
continue
optional[field] = schema
details = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": op_id,
"source_type": source_type,
"target_type": target_type,
"optional_args": optional,
"is_internal_operation": is_internal,
}
result = CreateValueFromDetails.create_operation_details(**details)
return result
check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/included_core_operations/create_from.py
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[CreateValueFromDetails, None]:
if not isinstance(module, CreateFromModule):
return None
source_type = None
for field_name, schema in module.inputs_schema.items():
if field_name == schema.type:
if source_type is not None:
logger.debug(
"ignore.operation",
operation_type="create_from",
reason=f"more than one possible target type field: {field_name}",
)
return None
source_type = field_name
if source_type is None:
return None
target_type = None
for field_name, schema in module.outputs_schema.items():
if field_name == schema.type:
if target_type is not None:
logger.debug(
"ignore.operation",
operation_type="create_from",
reason=f"more than one possible target type field: {field_name}",
)
return None
target_type = field_name
if target_type is None:
return None
op_id = self._calculate_op_id(source_type=source_type, target_type=target_type)
if (
"any" in self._kiara.type_registry.get_type_lineage(target_type)
and target_type != "any"
):
is_internal = False
else:
is_internal = True
optional = {}
for field, schema in module.inputs_schema.items():
if field == source_type:
continue
optional[field] = schema
details = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": op_id,
"source_type": source_type,
"target_type": target_type,
"optional_args": optional,
"is_internal_operation": is_internal,
}
result = CreateValueFromDetails.create_operation_details(**details)
return result
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/included_core_operations/create_from.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = {}
for name, module_cls in self._kiara.module_type_classes.items():
if not hasattr(module_cls, "retrieve_supported_create_combinations"):
continue
try:
supported_combinations = module_cls.retrieve_supported_create_combinations() # type: ignore
for sup_comb in supported_combinations:
source_type = sup_comb["source_type"]
target_type = sup_comb["target_type"]
func = sup_comb["func"]
if source_type not in self._kiara.data_type_names:
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Source type '{source_type}' not registered.",
)
continue
if target_type not in self._kiara.data_type_names:
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Target type '{target_type}' not registered.",
)
continue
if not hasattr(module_cls, func):
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Specified create function '{func}' not available.",
)
continue
mc = {"source_type": source_type, "target_type": target_type}
# TODO: check whether module config actually supports those, for now, only 'CreateFromModule' subtypes are supported
_func = getattr(module_cls, func)
doc = DocumentationMetadataModel.from_function(_func)
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
op_id = self._calculate_op_id(
source_type=source_type, target_type=target_type
)
result[op_id] = oc
except Exception as e:
log_exception(e)
logger.debug(
"ignore.create_operation_instance", module_type=name, reason=e
)
continue
return result.values()
CreateValueFromDetails (BaseOperationDetails)
pydantic-model
¶Source code in kiara/operations/included_core_operations/create_from.py
class CreateValueFromDetails(BaseOperationDetails):
source_type: str = Field(description="The type of the value to be created.")
target_type: str = Field(description="The result type.")
optional_args: Mapping[str, ValueSchema] = Field(description="Optional arguments.")
# def retrieve_inputs_schema(self) -> ValueSetSchema:
#
# result: Dict[str, Union[ValueSchema, Dict[str, Any]]] = {
# self.source_type: {"type": self.source_type, "doc": "The source value."},
# }
# for field, schema in self.optional_args.items():
# if field in result.keys():
# raise Exception(
# f"Can't create 'create_from' operation '{self.source_type}' -> '{self.target_type}': duplicate input field '{field}'."
# )
# result[field] = schema
# return result
#
# def retrieve_outputs_schema(self) -> ValueSetSchema:
#
# return {
# self.target_type: {"type": self.target_type, "doc": "The result value."}
# }
export_as
¶logger
¶
ExportAsOperationDetails (BaseOperationDetails)
pydantic-model
¶Source code in kiara/operations/included_core_operations/export_as.py
class ExportAsOperationDetails(BaseOperationDetails):
source_type: str = Field(description="The type of the value to be created.")
target_profile: str = Field(description="The result profile type.")
optional_args: Mapping[str, ValueSchema] = Field(description="Optional arguments.")
# def retrieve_inputs_schema(self) -> ValueSetSchema:
#
# result: Dict[str, Union[ValueSchema, Dict[str, Any]]] = {
# self.source_type: {"type": self.source_type, "doc": "The source value."},
# }
# for field, schema in self.optional_args.items():
# if field in result.keys():
# raise Exception(
# f"Can't create 'create_from' operation '{self.source_type}' -> '{self.target_profile}': duplicate input field '{field}'."
# )
# result[field] = schema
# return result
#
# def retrieve_outputs_schema(self) -> ValueSetSchema:
#
# return {
# "export_details": {
# "type": "dict",
# "doc": "Details about the exported data/files.",
# }
# }
ExportAsOperationType (OperationType)
¶Source code in kiara/operations/included_core_operations/export_as.py
class ExportAsOperationType(OperationType[ExportAsOperationDetails]):
_operation_type_name = "export_as"
def _calculate_op_id(self, source_type: str, target_profile: str):
if source_type == "any":
operation_id = f"export.as.{target_profile}"
else:
operation_id = f"export.{source_type}.as.{target_profile}"
return operation_id
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = []
for name, module_cls in self._kiara.module_type_classes.items():
if not hasattr(module_cls, "retrieve_supported_export_combinations"):
continue
try:
supported_combinations = module_cls.retrieve_supported_export_combinations() # type: ignore
for sup_comb in supported_combinations:
source_type = sup_comb["source_type"]
target_profile = sup_comb["target_profile"]
func = sup_comb["func"]
if source_type not in self._kiara.data_type_names:
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Source type '{source_type}' not registered.",
)
continue
if not hasattr(module_cls, func):
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Specified create function '{func}' not available.",
)
continue
mc = {"source_type": source_type, "target_profile": target_profile}
# TODO: check whether module config actually supports those, for now, only 'DataExportModule' subtypes are supported
_func = getattr(module_cls, func)
doc = DocumentationMetadataModel.from_function(_func)
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
result.append(oc)
except Exception as e:
log_exception(e)
logger.debug(
"ignore.create_operation_instance", module_type=name, reason=e
)
continue
return result
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[ExportAsOperationDetails, None]:
if not isinstance(module, DataExportModule):
return None
source_type = None
for field_name, schema in module.inputs_schema.items():
if field_name == schema.type:
if source_type is not None:
logger.debug(
"ignore.operation",
operation_type="create_from",
reason=f"more than one possible target type field: {field_name}",
)
return None
source_type = field_name
if source_type is None:
return None
target_profile = module.config.target_profile
op_id = self._calculate_op_id(
source_type=source_type, target_profile=target_profile
)
optional = {}
for field, schema in module.inputs_schema.items():
if field in [source_type]:
continue
optional[field] = schema
details = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": op_id,
"source_type": source_type,
"target_profile": target_profile,
"optional_args": optional,
"is_internal_operation": False,
}
result = ExportAsOperationDetails.create_operation_details(**details)
return result
check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/included_core_operations/export_as.py
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[ExportAsOperationDetails, None]:
if not isinstance(module, DataExportModule):
return None
source_type = None
for field_name, schema in module.inputs_schema.items():
if field_name == schema.type:
if source_type is not None:
logger.debug(
"ignore.operation",
operation_type="create_from",
reason=f"more than one possible target type field: {field_name}",
)
return None
source_type = field_name
if source_type is None:
return None
target_profile = module.config.target_profile
op_id = self._calculate_op_id(
source_type=source_type, target_profile=target_profile
)
optional = {}
for field, schema in module.inputs_schema.items():
if field in [source_type]:
continue
optional[field] = schema
details = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": op_id,
"source_type": source_type,
"target_profile": target_profile,
"optional_args": optional,
"is_internal_operation": False,
}
result = ExportAsOperationDetails.create_operation_details(**details)
return result
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/included_core_operations/export_as.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = []
for name, module_cls in self._kiara.module_type_classes.items():
if not hasattr(module_cls, "retrieve_supported_export_combinations"):
continue
try:
supported_combinations = module_cls.retrieve_supported_export_combinations() # type: ignore
for sup_comb in supported_combinations:
source_type = sup_comb["source_type"]
target_profile = sup_comb["target_profile"]
func = sup_comb["func"]
if source_type not in self._kiara.data_type_names:
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Source type '{source_type}' not registered.",
)
continue
if not hasattr(module_cls, func):
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Specified create function '{func}' not available.",
)
continue
mc = {"source_type": source_type, "target_profile": target_profile}
# TODO: check whether module config actually supports those, for now, only 'DataExportModule' subtypes are supported
_func = getattr(module_cls, func)
doc = DocumentationMetadataModel.from_function(_func)
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
result.append(oc)
except Exception as e:
log_exception(e)
logger.debug(
"ignore.create_operation_instance", module_type=name, reason=e
)
continue
return result
filter
¶logger
¶
FilterOperationDetails (BaseOperationDetails)
pydantic-model
¶Source code in kiara/operations/included_core_operations/filter.py
class FilterOperationDetails(BaseOperationDetails):
data_type: str = Field(description="The data type of the value to be filtered.")
data_type_config: Mapping[str, Any] = Field(
description="The configuration of the data type to be filtered.",
default_factory=dict,
)
filter_name: str = Field(description="The filter operation name.")
optional_args: Mapping[str, ValueSchema] = Field(description="Optional arguments.")
# def retrieve_inputs_schema(self) -> ValueSetSchema:
#
# result: Dict[str, Union[ValueSchema, Dict[str, Any]]] = {
# self.data_type: {
# "type": self.data_type,
# "type_config": self.data_type_config,
# "doc": "The value.",
# },
# }
# for field, schema in self.optional_args.items():
# if field in result.keys():
# raise Exception(
# f"Can't create 'filter' operation '{self.filter_name}': duplicate input field '{field}'."
# )
# result[field] = schema
# return result
#
# def retrieve_outputs_schema(self) -> ValueSetSchema:
#
# return {
# self.data_type: {
# "type": self.data_type,
# "type_config": self.data_type_config,
# "doc": "Details about the exported data/files.",
# },
# }
# def create_module_inputs(self, inputs: Mapping[str, Any]) -> Mapping[str, Any]:
#
# _inputs = dict(inputs)
# v = _inputs.pop("value")
# assert self.data_type not in _inputs.keys()
# _inputs[self.data_type] = v
# return _inputs
#
# def create_operation_outputs(self, outputs: ValueMap) -> Mapping[str, Value]:
#
# _outputs = dict(outputs)
# v = _outputs.pop(self.data_type)
# assert "value" not in _outputs.keys()
# _outputs["value"] = v
# return _outputs
data_type: str
pydantic-field
required
¶The data type of the value to be filtered.
data_type_config: Mapping[str, Any]
pydantic-field
¶The configuration of the data type to be filtered.
filter_name: str
pydantic-field
required
¶The filter operation name.
optional_args: Mapping[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶Optional arguments.
FilterOperationType (OperationType)
¶Source code in kiara/operations/included_core_operations/filter.py
class FilterOperationType(OperationType[FilterOperationDetails]):
_operation_type_name = "filter"
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = []
for name, module_cls in self._kiara.module_type_classes.items():
if not issubclass(module_cls, FilterModule):
continue
try:
data_type_data = module_cls.get_supported_type()
data_type: str = data_type_data["type"] # type: ignore
# data_type_config: Mapping[str, Any] = data_type["type_config"] # type: ignore
# TODO; try to create data type obj?
if data_type not in self._kiara.data_type_names:
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Data type '{data_type}' not registered.",
)
continue
supported_filters = module_cls.get_supported_filters()
for filter in supported_filters:
func_name = f"filter__{filter}"
if not hasattr(module_cls, func_name):
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Specified filter function '{func_name}' not available.",
)
continue
mc = {"filter_name": filter}
# TODO: check whether module config actually supports those, for now, only 'DataExportModule' subtypes are supported
_func = getattr(module_cls, func_name)
doc = DocumentationMetadataModel.from_function(_func)
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
result.append(oc)
except Exception as e:
log_exception(e)
logger.debug(
"ignore.create_operation_instance", module_type=name, reason=e
)
continue
return result
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[FilterOperationDetails, None]:
if not isinstance(module, FilterModule):
return None
data_type_data = module.__class__.get_supported_type()
data_type: str = data_type_data["type"] # type: ignore
data_type_config: Mapping[str, Any] = data_type_data["type_config"] # type: ignore
filter_name = module.get_config_value("filter_name")
op_id = f"{data_type}_filter.{filter_name}"
optional = {}
for field, schema in module.inputs_schema.items():
if field in [data_type]:
continue
optional[field] = schema
details = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": op_id,
"data_type": data_type,
"data_type_config": data_type_config,
"filter_name": filter_name,
"optional_args": optional,
"is_internal_operation": False,
}
result = FilterOperationDetails.create_operation_details(**details)
return result
# def find_filter_operations_for_data_type(
# self, data_type: str
# ) -> Dict[str, Operation]:
#
# result = {}
# for op in self.operations.values():
# details: FilterOperationDetails = op.operation_details # type: ignore
# if details.data_type == data_type:
# result[details.filter_name] = op
#
# return result
def get_filter(self, data_type: str, filter_name: str) -> Filter:
try:
op = self._kiara.operation_registry.get_operation(operation_id=filter_name)
except Exception:
op_id = f"{data_type}_filter.{filter_name}"
op = self.operations.get(op_id, None) # type: ignore
if op is None:
raise Exception(
f"No filter operation '{filter_name}' available for type '{data_type}'."
)
inp_match = []
for input_name, schema in op.inputs_schema.items():
# TODO: check lineage/profiles
if schema.type == data_type:
inp_match.append(input_name)
if not inp_match:
raise Exception(
f"Can't retrieve filter with name '{filter_name}' for data type: '{data_type}': input fields for operation '{op.operation_id}' don't match."
)
if len(inp_match) > 1:
if "value" in inp_match:
inp_match = ["value"]
elif data_type in inp_match:
inp_match = [data_type]
else:
raise Exception(
f"Can't retrieve filter with name '{filter_name}' for data type: '{data_type}', operation '{op.operation_id}' has multiple potential input fields: {', '.join(inp_match)}."
)
input_field = inp_match[0]
outp_match = []
for output_name, schema in op.outputs_schema.items():
# TODO: check lineage/profiles
if schema.type == data_type:
outp_match.append(output_name)
if not outp_match:
raise Exception(
f"Can't retrieve filter with name '{filter_name}' for data type: '{data_type}': output fields for operation '{op.operation_id}' don't match."
)
if len(outp_match) > 1:
if "value" in outp_match:
outp_match = ["value"]
elif data_type in outp_match:
outp_match = [data_type]
else:
raise Exception(
f"Can't retrieve filter with name '{filter_name}' for data type: '{data_type}', operation '{op.operation_id}' has multiple potential output fields: {', '.join(outp_match)}."
)
output_field = outp_match[0]
filter = Filter.construct(
operation=op,
input_name=input_field,
output_name=output_field,
data_type=data_type,
)
return filter
def assemble_filter_pipeline_config(
self,
data_type: str,
filters: Union[str, Iterable[str], Mapping[str, str]],
endpoint: Union[None, Manifest, str] = None,
endpoint_input_field: Union[str, None] = None,
endpoint_step_id: Union[str, None] = None,
extra_input_aliases: Union[None, Mapping[str, str]] = None,
extra_output_aliases: Union[None, Mapping[str, str]] = None,
) -> PipelineConfig:
"""Assemble a (pipeline) module config to filter values of a specific data type.
Optionally, a module that uses the filtered dataset as input can be specified.
# TODO: document filter names
For the 'filters' argument, the accepted inputs are:
- a string, in which case a single-step pipeline will be created, with the string referencing the operation id or filter
- a list of strings: in which case a multi-step pipeline will be created, the step_ids will be calculated automatically
- a map of string pairs: the keys are step ids, the values operation ids or filter names
Arguments:
data_type: the type of the data to filter
filters: a list of operation ids or filter names (and potentiall step_ids if type is a mapping)
endpoint: optional module to put as last step in the created pipeline
endpoing_input_field: field name of the input that will receive the filtered value
endpoint_step_id: id to use for the endpoint step (module type name will be used if not provided)
extra_output_aliases: extra output aliases to add to the pipeline config
Returns:
the (pipeline) module configuration of the filter pipeline
"""
steps: List[Mapping[str, Any]] = []
last_filter_id: Union[str, None] = None
last_filter_output_name: Union[str, None] = None
input_aliases: Dict[str, str] = {}
output_aliases: Dict[str, str] = {}
if isinstance(filters, str):
filters = {filters: filters}
if not isinstance(filters, Mapping):
_filters = {}
_step_ids: List[str] = []
for filter_name in filters:
step_id = find_free_id(stem=filter_name, current_ids=_step_ids)
_filters[step_id] = filter_name
filters = _filters
for filter_name, step_id in filters.items():
if not input_aliases:
input_aliases[f"{filter_name}.value"] = "value"
filter = self.get_filter(data_type=data_type, filter_name=filter_name)
step_data = {
"module_type": filter.operation.operation_id,
"step_id": step_id,
}
if last_filter_id:
step_data["input_links"] = {
filter.input_name: f"{last_filter_id}.{last_filter_output_name}"
}
last_filter_id = step_id
last_filter_output_name = filter.output_name
steps.append(step_data)
output_aliases[f"{step_id}.value"] = f"{step_id}__filtered"
output_aliases[f"{last_filter_id}.{last_filter_output_name}"] = "filtered_value"
doc = f"Auto generated filter operation ({'->'.join(filters.keys())}) for type '{data_type}'"
if endpoint:
endpoint_module = self._kiara.module_registry.create_module(
manifest=endpoint
)
if endpoint_input_field is None:
matches = []
for field_name, schema in endpoint_module.inputs_schema.items():
# TODO: check profiles/lineage
if schema.type == data_type:
matches.append(field_name)
if not matches:
raise Exception(
f"Can't assemble filter operation: no potential input field of type {data_type} for endpoint module found."
)
elif len(matches) > 1:
raise Exception(
f"Can't assemble filter operation: multiple potential input fields of type {data_type} for endpoint module found: {', '.join(matches)}"
)
endpoint_input_field = matches[0]
if not endpoint_step_id:
endpoint_step_id = find_free_id(
stem=endpoint_module.module_type_name, current_ids=filters.values()
)
step_data = {
"module_type": endpoint_module.module_type_name,
"module_config": endpoint_module.config.dict(),
"step_id": endpoint_step_id,
}
step_data["input_links"] = {
endpoint_input_field: {f"{last_filter_id}.{last_filter_output_name}"}
}
# for field_name in endpoint_module.output_names:
# output_aliases[f"{endpoint_step_id}.{field_name}"] = f"endpoint__{field_name}"
doc = f"{doc}, feeding into endpoing module '{endpoint_module.module_type_name}'."
steps.append(step_data)
else:
doc = f"{doc}."
if extra_output_aliases:
for k, v in extra_output_aliases.items():
output_aliases[k] = v
if extra_input_aliases:
input_aliases.update(extra_input_aliases)
# raise NotImplementedError("Extra input aliases not supported yet.")
pipeline_config = PipelineConfig.from_config(
pipeline_name="_filter_pipeline",
data={
"steps": steps,
"input_aliases": input_aliases,
"output_aliases": output_aliases,
"doc": doc,
},
)
return pipeline_config
def create_filter_operation(
self,
data_type: str,
filters: Union[Iterable[str], Mapping[str, str]],
endpoint: Union[None, Manifest, str] = None,
endpoint_input_field: Union[str, None] = None,
endpoint_step_id: Union[str, None] = None,
) -> Operation:
pipeline_config = self.assemble_filter_pipeline_config(
data_type=data_type,
filters=filters,
endpoint=endpoint,
endpoint_input_field=endpoint_input_field,
endpoint_step_id=endpoint_step_id,
)
manifest = Manifest(
module_type="pipeline", module_config=pipeline_config.dict()
)
module = self._kiara.create_module(manifest=manifest)
op_details = PipelineOperationDetails.create_operation_details(
operation_id=module.config.pipeline_name,
pipeline_inputs_schema=module.inputs_schema,
pipeline_outputs_schema=module.outputs_schema,
pipeline_config=module.config,
)
operation = Operation(
module_type=manifest.module_type,
module_config=manifest.module_config,
operation_id=op_details.operation_id,
operation_details=op_details,
module_details=KiaraModuleInstance.from_module(module),
metadata={},
doc=pipeline_config.doc,
)
return operation
assemble_filter_pipeline_config(self, data_type, filters, endpoint=None, endpoint_input_field=None, endpoint_step_id=None, extra_input_aliases=None, extra_output_aliases=None)
¶Assemble a (pipeline) module config to filter values of a specific data type.
Optionally, a module that uses the filtered dataset as input can be specified.
TODO: document filter names¶
For the 'filters' argument, the accepted inputs are: - a string, in which case a single-step pipeline will be created, with the string referencing the operation id or filter - a list of strings: in which case a multi-step pipeline will be created, the step_ids will be calculated automatically - a map of string pairs: the keys are step ids, the values operation ids or filter names
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data_type |
str |
the type of the data to filter |
required |
filters |
Union[str, Iterable[str], Mapping[str, str]] |
a list of operation ids or filter names (and potentiall step_ids if type is a mapping) |
required |
endpoint |
Union[NoneType, kiara.models.module.manifest.Manifest, str] |
optional module to put as last step in the created pipeline |
None |
endpoing_input_field |
field name of the input that will receive the filtered value |
required | |
endpoint_step_id |
Optional[str] |
id to use for the endpoint step (module type name will be used if not provided) |
None |
extra_output_aliases |
Optional[Mapping[str, str]] |
extra output aliases to add to the pipeline config |
None |
Returns:
| Type | Description |
|---|---|
PipelineConfig |
the (pipeline) module configuration of the filter pipeline |
Source code in kiara/operations/included_core_operations/filter.py
def assemble_filter_pipeline_config(
self,
data_type: str,
filters: Union[str, Iterable[str], Mapping[str, str]],
endpoint: Union[None, Manifest, str] = None,
endpoint_input_field: Union[str, None] = None,
endpoint_step_id: Union[str, None] = None,
extra_input_aliases: Union[None, Mapping[str, str]] = None,
extra_output_aliases: Union[None, Mapping[str, str]] = None,
) -> PipelineConfig:
"""Assemble a (pipeline) module config to filter values of a specific data type.
Optionally, a module that uses the filtered dataset as input can be specified.
# TODO: document filter names
For the 'filters' argument, the accepted inputs are:
- a string, in which case a single-step pipeline will be created, with the string referencing the operation id or filter
- a list of strings: in which case a multi-step pipeline will be created, the step_ids will be calculated automatically
- a map of string pairs: the keys are step ids, the values operation ids or filter names
Arguments:
data_type: the type of the data to filter
filters: a list of operation ids or filter names (and potentiall step_ids if type is a mapping)
endpoint: optional module to put as last step in the created pipeline
endpoing_input_field: field name of the input that will receive the filtered value
endpoint_step_id: id to use for the endpoint step (module type name will be used if not provided)
extra_output_aliases: extra output aliases to add to the pipeline config
Returns:
the (pipeline) module configuration of the filter pipeline
"""
steps: List[Mapping[str, Any]] = []
last_filter_id: Union[str, None] = None
last_filter_output_name: Union[str, None] = None
input_aliases: Dict[str, str] = {}
output_aliases: Dict[str, str] = {}
if isinstance(filters, str):
filters = {filters: filters}
if not isinstance(filters, Mapping):
_filters = {}
_step_ids: List[str] = []
for filter_name in filters:
step_id = find_free_id(stem=filter_name, current_ids=_step_ids)
_filters[step_id] = filter_name
filters = _filters
for filter_name, step_id in filters.items():
if not input_aliases:
input_aliases[f"{filter_name}.value"] = "value"
filter = self.get_filter(data_type=data_type, filter_name=filter_name)
step_data = {
"module_type": filter.operation.operation_id,
"step_id": step_id,
}
if last_filter_id:
step_data["input_links"] = {
filter.input_name: f"{last_filter_id}.{last_filter_output_name}"
}
last_filter_id = step_id
last_filter_output_name = filter.output_name
steps.append(step_data)
output_aliases[f"{step_id}.value"] = f"{step_id}__filtered"
output_aliases[f"{last_filter_id}.{last_filter_output_name}"] = "filtered_value"
doc = f"Auto generated filter operation ({'->'.join(filters.keys())}) for type '{data_type}'"
if endpoint:
endpoint_module = self._kiara.module_registry.create_module(
manifest=endpoint
)
if endpoint_input_field is None:
matches = []
for field_name, schema in endpoint_module.inputs_schema.items():
# TODO: check profiles/lineage
if schema.type == data_type:
matches.append(field_name)
if not matches:
raise Exception(
f"Can't assemble filter operation: no potential input field of type {data_type} for endpoint module found."
)
elif len(matches) > 1:
raise Exception(
f"Can't assemble filter operation: multiple potential input fields of type {data_type} for endpoint module found: {', '.join(matches)}"
)
endpoint_input_field = matches[0]
if not endpoint_step_id:
endpoint_step_id = find_free_id(
stem=endpoint_module.module_type_name, current_ids=filters.values()
)
step_data = {
"module_type": endpoint_module.module_type_name,
"module_config": endpoint_module.config.dict(),
"step_id": endpoint_step_id,
}
step_data["input_links"] = {
endpoint_input_field: {f"{last_filter_id}.{last_filter_output_name}"}
}
# for field_name in endpoint_module.output_names:
# output_aliases[f"{endpoint_step_id}.{field_name}"] = f"endpoint__{field_name}"
doc = f"{doc}, feeding into endpoing module '{endpoint_module.module_type_name}'."
steps.append(step_data)
else:
doc = f"{doc}."
if extra_output_aliases:
for k, v in extra_output_aliases.items():
output_aliases[k] = v
if extra_input_aliases:
input_aliases.update(extra_input_aliases)
# raise NotImplementedError("Extra input aliases not supported yet.")
pipeline_config = PipelineConfig.from_config(
pipeline_name="_filter_pipeline",
data={
"steps": steps,
"input_aliases": input_aliases,
"output_aliases": output_aliases,
"doc": doc,
},
)
return pipeline_config
check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/included_core_operations/filter.py
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[FilterOperationDetails, None]:
if not isinstance(module, FilterModule):
return None
data_type_data = module.__class__.get_supported_type()
data_type: str = data_type_data["type"] # type: ignore
data_type_config: Mapping[str, Any] = data_type_data["type_config"] # type: ignore
filter_name = module.get_config_value("filter_name")
op_id = f"{data_type}_filter.{filter_name}"
optional = {}
for field, schema in module.inputs_schema.items():
if field in [data_type]:
continue
optional[field] = schema
details = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": op_id,
"data_type": data_type,
"data_type_config": data_type_config,
"filter_name": filter_name,
"optional_args": optional,
"is_internal_operation": False,
}
result = FilterOperationDetails.create_operation_details(**details)
return result
create_filter_operation(self, data_type, filters, endpoint=None, endpoint_input_field=None, endpoint_step_id=None)
¶Source code in kiara/operations/included_core_operations/filter.py
def create_filter_operation(
self,
data_type: str,
filters: Union[Iterable[str], Mapping[str, str]],
endpoint: Union[None, Manifest, str] = None,
endpoint_input_field: Union[str, None] = None,
endpoint_step_id: Union[str, None] = None,
) -> Operation:
pipeline_config = self.assemble_filter_pipeline_config(
data_type=data_type,
filters=filters,
endpoint=endpoint,
endpoint_input_field=endpoint_input_field,
endpoint_step_id=endpoint_step_id,
)
manifest = Manifest(
module_type="pipeline", module_config=pipeline_config.dict()
)
module = self._kiara.create_module(manifest=manifest)
op_details = PipelineOperationDetails.create_operation_details(
operation_id=module.config.pipeline_name,
pipeline_inputs_schema=module.inputs_schema,
pipeline_outputs_schema=module.outputs_schema,
pipeline_config=module.config,
)
operation = Operation(
module_type=manifest.module_type,
module_config=manifest.module_config,
operation_id=op_details.operation_id,
operation_details=op_details,
module_details=KiaraModuleInstance.from_module(module),
metadata={},
doc=pipeline_config.doc,
)
return operation
get_filter(self, data_type, filter_name)
¶Source code in kiara/operations/included_core_operations/filter.py
def get_filter(self, data_type: str, filter_name: str) -> Filter:
try:
op = self._kiara.operation_registry.get_operation(operation_id=filter_name)
except Exception:
op_id = f"{data_type}_filter.{filter_name}"
op = self.operations.get(op_id, None) # type: ignore
if op is None:
raise Exception(
f"No filter operation '{filter_name}' available for type '{data_type}'."
)
inp_match = []
for input_name, schema in op.inputs_schema.items():
# TODO: check lineage/profiles
if schema.type == data_type:
inp_match.append(input_name)
if not inp_match:
raise Exception(
f"Can't retrieve filter with name '{filter_name}' for data type: '{data_type}': input fields for operation '{op.operation_id}' don't match."
)
if len(inp_match) > 1:
if "value" in inp_match:
inp_match = ["value"]
elif data_type in inp_match:
inp_match = [data_type]
else:
raise Exception(
f"Can't retrieve filter with name '{filter_name}' for data type: '{data_type}', operation '{op.operation_id}' has multiple potential input fields: {', '.join(inp_match)}."
)
input_field = inp_match[0]
outp_match = []
for output_name, schema in op.outputs_schema.items():
# TODO: check lineage/profiles
if schema.type == data_type:
outp_match.append(output_name)
if not outp_match:
raise Exception(
f"Can't retrieve filter with name '{filter_name}' for data type: '{data_type}': output fields for operation '{op.operation_id}' don't match."
)
if len(outp_match) > 1:
if "value" in outp_match:
outp_match = ["value"]
elif data_type in outp_match:
outp_match = [data_type]
else:
raise Exception(
f"Can't retrieve filter with name '{filter_name}' for data type: '{data_type}', operation '{op.operation_id}' has multiple potential output fields: {', '.join(outp_match)}."
)
output_field = outp_match[0]
filter = Filter.construct(
operation=op,
input_name=input_field,
output_name=output_field,
data_type=data_type,
)
return filter
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/included_core_operations/filter.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = []
for name, module_cls in self._kiara.module_type_classes.items():
if not issubclass(module_cls, FilterModule):
continue
try:
data_type_data = module_cls.get_supported_type()
data_type: str = data_type_data["type"] # type: ignore
# data_type_config: Mapping[str, Any] = data_type["type_config"] # type: ignore
# TODO; try to create data type obj?
if data_type not in self._kiara.data_type_names:
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Data type '{data_type}' not registered.",
)
continue
supported_filters = module_cls.get_supported_filters()
for filter in supported_filters:
func_name = f"filter__{filter}"
if not hasattr(module_cls, func_name):
logger.debug(
"ignore.operation_config",
module_type=name,
reason=f"Specified filter function '{func_name}' not available.",
)
continue
mc = {"filter_name": filter}
# TODO: check whether module config actually supports those, for now, only 'DataExportModule' subtypes are supported
_func = getattr(module_cls, func_name)
doc = DocumentationMetadataModel.from_function(_func)
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
result.append(oc)
except Exception as e:
log_exception(e)
logger.debug(
"ignore.create_operation_instance", module_type=name, reason=e
)
continue
return result
metadata
¶
ExtractMetadataDetails (BaseOperationDetails)
pydantic-model
¶A model that contains information needed to describe an 'extract_metadata' operation.
Source code in kiara/operations/included_core_operations/metadata.py
class ExtractMetadataDetails(BaseOperationDetails):
"""A model that contains information needed to describe an 'extract_metadata' operation."""
data_type: str = Field(
description="The data type this metadata operation can be used with."
)
metadata_key: str = Field(description="The metadata key.")
input_field_name: str = Field(description="The input field name.")
result_field_name: str = Field(description="The result field name.")
# def retrieve_inputs_schema(self) -> ValueSetSchema:
# return {
# "value": {
# "type": self.data_type,
# "doc": f"The {self.data_type} value to extract metadata from.",
# }
# }
#
# def retrieve_outputs_schema(self) -> ValueSetSchema:
#
# return {"value_metadata": {"type": "value_metadata", "doc": "The metadata."}}
#
# def create_module_inputs(self, inputs: Mapping[str, Any]) -> Mapping[str, Any]:
# return {self.input_field_name: inputs["value"]}
#
# def create_operation_outputs(self, outputs: ValueMap) -> ValueMap:
#
# return outputs
data_type: str
pydantic-field
required
¶The data type this metadata operation can be used with.
input_field_name: str
pydantic-field
required
¶The input field name.
metadata_key: str
pydantic-field
required
¶The metadata key.
result_field_name: str
pydantic-field
required
¶The result field name.
ExtractMetadataOperationType (OperationType)
¶An operation that extracts metadata of a specific type from value data.
For a module profile to be picked up by this operation type, it needs to have: - exactly one input field - that input field must have the same name as its value type, or be 'value' - exactly one output field, whose field name is called 'value_metadata', and where the value has the type 'internal_model'
Source code in kiara/operations/included_core_operations/metadata.py
class ExtractMetadataOperationType(OperationType[ExtractMetadataDetails]):
"""An operation that extracts metadata of a specific type from value data.
For a module profile to be picked up by this operation type, it needs to have:
- exactly one input field
- that input field must have the same name as its value type, or be 'value'
- exactly one output field, whose field name is called 'value_metadata', and where the value has the type 'internal_model'
"""
_operation_type_name = "extract_metadata"
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
model_registry = ModelRegistry.instance()
all_models = model_registry.get_models_of_type(ValueMetadata)
result = []
for model_id, model_cls_info in all_models.item_infos.items():
model_cls: Type[ValueMetadata] = model_cls_info.python_class.get_class() # type: ignore
metadata_key = model_cls._metadata_key # type: ignore
data_types = model_cls.retrieve_supported_data_types()
if isinstance(data_types, str):
data_types = [data_types]
for data_type in data_types:
config = {
"module_type": "value.extract_metadata",
"module_config": {
"data_type": data_type,
"kiara_model_id": model_cls._kiara_model_id, # type: ignore
},
"doc": f"Extract '{metadata_key}' metadata for value type '{data_type}'.",
}
result.append(config)
return result
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[ExtractMetadataDetails, None]:
if len(module.outputs_schema) != 1:
return None
if (
"value_metadata" not in module.outputs_schema
or module.outputs_schema["value_metadata"].type != "internal_model"
):
return None
if len(module.inputs_schema) != 1:
return None
input_field_name = next(iter(module.inputs_schema.keys()))
input_schema = module.inputs_schema.get(input_field_name)
assert input_schema is not None
if input_field_name != input_schema.type and input_field_name != "value":
return None
data_type_name = module.inputs_schema["value"].type
model_id: str = module.get_config_value("kiara_model_id")
registry = ModelRegistry.instance()
metadata_model_cls = registry.get_model_cls(
kiara_model_id=model_id, required_subclass=ValueMetadata
)
metadata_key = metadata_model_cls._metadata_key # type: ignore
if data_type_name == "any":
op_id = f"extract.{metadata_key}.metadata"
else:
op_id = f"extract.{metadata_key}.metadata.from.{data_type_name}"
details = ExtractMetadataDetails.create_operation_details(
module_inputs_schema=module.inputs_schema,
module_outputs_schema=module.outputs_schema,
operation_id=op_id,
data_type=data_type_name,
metadata_key=metadata_key,
input_field_name=input_field_name,
result_field_name="value_metadata",
is_internal_operation=True,
)
return details
def get_operations_for_data_type(self, data_type: str) -> Mapping[str, Operation]:
"""Return all available metadata extract operations for the provided type (and it's parent types).
Arguments:
data_type: the value type
Returns:
a mapping with the metadata type as key, and the operation as value
"""
lineage = set(
self._kiara.type_registry.get_type_lineage(data_type_name=data_type)
)
result = {}
for op_id, op in self.operations.items():
op_details = self.retrieve_operation_details(op)
included = op_details.data_type in lineage
if not included:
continue
metadata_key = op_details.metadata_key
if metadata_key in result:
raise Exception(
f"Duplicate metadata operations for type '{metadata_key}'."
)
result[metadata_key] = op
return result
check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/included_core_operations/metadata.py
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[ExtractMetadataDetails, None]:
if len(module.outputs_schema) != 1:
return None
if (
"value_metadata" not in module.outputs_schema
or module.outputs_schema["value_metadata"].type != "internal_model"
):
return None
if len(module.inputs_schema) != 1:
return None
input_field_name = next(iter(module.inputs_schema.keys()))
input_schema = module.inputs_schema.get(input_field_name)
assert input_schema is not None
if input_field_name != input_schema.type and input_field_name != "value":
return None
data_type_name = module.inputs_schema["value"].type
model_id: str = module.get_config_value("kiara_model_id")
registry = ModelRegistry.instance()
metadata_model_cls = registry.get_model_cls(
kiara_model_id=model_id, required_subclass=ValueMetadata
)
metadata_key = metadata_model_cls._metadata_key # type: ignore
if data_type_name == "any":
op_id = f"extract.{metadata_key}.metadata"
else:
op_id = f"extract.{metadata_key}.metadata.from.{data_type_name}"
details = ExtractMetadataDetails.create_operation_details(
module_inputs_schema=module.inputs_schema,
module_outputs_schema=module.outputs_schema,
operation_id=op_id,
data_type=data_type_name,
metadata_key=metadata_key,
input_field_name=input_field_name,
result_field_name="value_metadata",
is_internal_operation=True,
)
return details
get_operations_for_data_type(self, data_type)
¶Return all available metadata extract operations for the provided type (and it's parent types).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data_type |
str |
the value type |
required |
Returns:
| Type | Description |
|---|---|
Mapping[str, kiara.models.module.operation.Operation] |
a mapping with the metadata type as key, and the operation as value |
Source code in kiara/operations/included_core_operations/metadata.py
def get_operations_for_data_type(self, data_type: str) -> Mapping[str, Operation]:
"""Return all available metadata extract operations for the provided type (and it's parent types).
Arguments:
data_type: the value type
Returns:
a mapping with the metadata type as key, and the operation as value
"""
lineage = set(
self._kiara.type_registry.get_type_lineage(data_type_name=data_type)
)
result = {}
for op_id, op in self.operations.items():
op_details = self.retrieve_operation_details(op)
included = op_details.data_type in lineage
if not included:
continue
metadata_key = op_details.metadata_key
if metadata_key in result:
raise Exception(
f"Duplicate metadata operations for type '{metadata_key}'."
)
result[metadata_key] = op
return result
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/included_core_operations/metadata.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
model_registry = ModelRegistry.instance()
all_models = model_registry.get_models_of_type(ValueMetadata)
result = []
for model_id, model_cls_info in all_models.item_infos.items():
model_cls: Type[ValueMetadata] = model_cls_info.python_class.get_class() # type: ignore
metadata_key = model_cls._metadata_key # type: ignore
data_types = model_cls.retrieve_supported_data_types()
if isinstance(data_types, str):
data_types = [data_types]
for data_type in data_types:
config = {
"module_type": "value.extract_metadata",
"module_config": {
"data_type": data_type,
"kiara_model_id": model_cls._kiara_model_id, # type: ignore
},
"doc": f"Extract '{metadata_key}' metadata for value type '{data_type}'.",
}
result.append(config)
return result
pipeline
¶logger
¶
PipelineOperationDetails (OperationDetails)
pydantic-model
¶Source code in kiara/operations/included_core_operations/pipeline.py
class PipelineOperationDetails(OperationDetails):
# @classmethod
# def create_from_module(cls, module: KiaraModule):
#
# return PipelineOperationDetails(
# operation_id=module.module_type_name,
# pipeline_inputs_schema=module.inputs_schema,
# pipeline_outputs_schema=module.outputs_schema,
# )
pipeline_inputs_schema: Mapping[str, ValueSchema] = Field(
description="The input schema for the pipeline."
)
pipeline_outputs_schema: Mapping[str, ValueSchema] = Field(
description="The output schema for the pipeline."
)
pipeline_config: PipelineConfig = Field(description="The pipeline config.")
_op_schema: OperationSchema = PrivateAttr(default=None)
def get_operation_schema(self) -> OperationSchema:
if self._op_schema is not None:
return self._op_schema
self._op_schema = OperationSchema(
alias=self.operation_id,
inputs_schema=self.pipeline_inputs_schema,
outputs_schema=self.pipeline_outputs_schema,
)
return self._op_schema
# def create_module_inputs(self, inputs: Mapping[str, Any]) -> Mapping[str, Any]:
# return inputs
#
# def create_operation_outputs(self, outputs: ValueMap) -> Mapping[str, Value]:
# return outputs
pipeline_config: PipelineConfig
pydantic-field
required
¶The pipeline config.
pipeline_inputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The input schema for the pipeline.
pipeline_outputs_schema: Mapping[str, kiara.models.values.value_schema.ValueSchema]
pydantic-field
required
¶The output schema for the pipeline.
get_operation_schema(self)
¶Source code in kiara/operations/included_core_operations/pipeline.py
def get_operation_schema(self) -> OperationSchema:
if self._op_schema is not None:
return self._op_schema
self._op_schema = OperationSchema(
alias=self.operation_id,
inputs_schema=self.pipeline_inputs_schema,
outputs_schema=self.pipeline_outputs_schema,
)
return self._op_schema
PipelineOperationType (OperationType)
¶Source code in kiara/operations/included_core_operations/pipeline.py
class PipelineOperationType(OperationType[PipelineOperationDetails]):
_operation_type_name = "pipeline"
def __init__(self, kiara: "Kiara", op_type_name: str):
super().__init__(kiara=kiara, op_type_name=op_type_name)
self._pipelines = None
@property
def pipeline_data(self):
if self._pipelines is not None:
return self._pipelines
ignore_errors = False
pipeline_paths: Dict[
str, Union[Mapping[str, Any], None]
] = find_all_kiara_pipeline_paths(skip_errors=ignore_errors)
for ep in self._kiara.context_config.extra_pipelines:
ep = os.path.realpath(ep)
if ep not in pipeline_paths.keys():
pipeline_paths[ep] = None
all_pipelines = []
for _path in pipeline_paths.keys():
path = Path(_path)
if not path.exists():
logger.warning(
"ignore.pipeline_path", path=path, reason="path does not exist"
)
continue
elif path.is_dir():
for root, dirnames, filenames in os.walk(path, topdown=True):
dirnames[:] = [d for d in dirnames if d not in DEFAULT_EXCLUDE_DIRS]
for filename in [
f
for f in filenames
if os.path.isfile(os.path.join(root, f))
and any(
f.endswith(ext) for ext in VALID_PIPELINE_FILE_EXTENSIONS
)
]:
full_path = os.path.join(root, filename)
try:
data = get_pipeline_details_from_path(path=full_path)
data = check_doc_sidecar(full_path, data)
existing_metadata = data.pop("metadata", {})
_md = pipeline_paths[_path]
if _md is None:
md = {}
else:
md = dict(_md)
md.update(existing_metadata)
data["metadata"] = md
all_pipelines.append(data)
except Exception as e:
log_exception(e)
logger.warning(
"ignore.pipeline_file", path=full_path, reason=str(e)
)
elif path.is_file():
data = get_pipeline_details_from_path(path=path)
data = check_doc_sidecar(path, data)
existing_metadata = data.pop("metadata", {})
_md = pipeline_paths[_path]
if _md is None:
md = {}
else:
md = dict(_md)
md.update(existing_metadata)
data["metadata"] = md
all_pipelines.append(data)
pipelines = {}
for pipeline in all_pipelines:
name = pipeline["data"].get("pipeline_name", None)
if name is None:
source = pipeline["source"]
name = os.path.basename(source)
if "." in name:
name, _ = name.rsplit(".", maxsplit=1)
pipeline["data"]["pipeline_name"] = name
pipelines[name] = pipeline
return pipelines
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
op_configs = []
for pipeline_name, pipeline_data in self.pipeline_data.items():
pipeline_config = dict(pipeline_data["data"])
pipeline_id = pipeline_config.pop("pipeline_name", None)
doc = pipeline_config.get("doc", None)
pipeline_metadata = pipeline_data["metadata"]
op_details = PipelineOperationConfig(
pipeline_name=pipeline_id,
pipeline_config=pipeline_config,
doc=doc,
metadata=pipeline_metadata,
)
op_configs.append(op_details)
return op_configs
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[PipelineOperationDetails, None]:
if isinstance(module, PipelineModule):
op_details = PipelineOperationDetails.create_operation_details(
operation_id=module.config.pipeline_name,
pipeline_inputs_schema=module.inputs_schema,
pipeline_outputs_schema=module.outputs_schema,
pipeline_config=module.config,
)
return op_details
else:
return None
pipeline_data
property
readonly
¶check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/included_core_operations/pipeline.py
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[PipelineOperationDetails, None]:
if isinstance(module, PipelineModule):
op_details = PipelineOperationDetails.create_operation_details(
operation_id=module.config.pipeline_name,
pipeline_inputs_schema=module.inputs_schema,
pipeline_outputs_schema=module.outputs_schema,
pipeline_config=module.config,
)
return op_details
else:
return None
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/included_core_operations/pipeline.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
op_configs = []
for pipeline_name, pipeline_data in self.pipeline_data.items():
pipeline_config = dict(pipeline_data["data"])
pipeline_id = pipeline_config.pop("pipeline_name", None)
doc = pipeline_config.get("doc", None)
pipeline_metadata = pipeline_data["metadata"]
op_details = PipelineOperationConfig(
pipeline_name=pipeline_id,
pipeline_config=pipeline_config,
doc=doc,
metadata=pipeline_metadata,
)
op_configs.append(op_details)
return op_configs
pretty_print
¶
PrettyPrintDetails (BaseOperationDetails)
pydantic-model
¶Source code in kiara/operations/included_core_operations/pretty_print.py
class PrettyPrintDetails(BaseOperationDetails):
source_type: str = Field(description="The type of the value to be rendered.")
target_type: str = Field(description="The type of the render result.")
# def retrieve_inputs_schema(self) -> ValueSetSchema:
#
# return {
# "value": {"type": "any", "doc": "The value to persist."},
# "render_type": {
# "type": "string",
# "doc": "The render target/type of render output.",
# },
# "render_config": {
# "type": "dict",
# "doc": "A value type specific configuration for how to render the data.",
# "optional": True,
# },
# }
#
# def retrieve_outputs_schema(self) -> ValueSetSchema:
#
# return {"rendered_value": {"type": "any", "doc": "The rendered value."}}
#
# def create_module_inputs(self, inputs: Mapping[str, Any]) -> Mapping[str, Any]:
#
# return {
# self.source_type: inputs["value"],
# "render_config": inputs.get("render_config", None),
# }
#
# def create_operation_outputs(self, outputs: ValueMap) -> Mapping[str, Value]:
# return {"rendered_value": outputs.get_value_obj("rendered_value")}
PrettyPrintOperationType (OperationType)
¶An operation that takes a value, and renders into a format that can be printed for output..
For a module profile to be picked up by this operation type, it needs to have: - exactly one output field named "rendered_value" - exactly two input fields, one of them named after the type it supports, and the other called 'render_config', of type 'dict'
Source code in kiara/operations/included_core_operations/pretty_print.py
class PrettyPrintOperationType(OperationType[PrettyPrintDetails]):
"""An operation that takes a value, and renders into a format that can be printed for output..
For a module profile to be picked up by this operation type, it needs to have:
- exactly one output field named "rendered_value"
- exactly two input fields, one of them named after the type it supports, and the other called 'render_config', of type 'dict'
"""
_operation_type_name = "pretty_print"
def _calculate_op_id(self, source_type: str, target_type: str):
if source_type == "any":
operation_id = f"pretty_print.as.{target_type}"
else:
operation_id = f"pretty_print.{source_type}.as.{target_type}"
return operation_id
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = {}
for name, module_cls in self._kiara.module_type_classes.items():
if not issubclass(module_cls, PrettyPrintModule):
continue
for (
source_type,
target_type,
) in module_cls.retrieve_supported_render_combinations():
if source_type not in self._kiara.data_type_names:
log_message("ignore.operation_config", operation_type="pretty_print", module_type=module_cls._module_type_name, source_type=source_type, target_type=target_type, reason=f"Source type '{source_type}' not registered.") # type: ignore
continue
if target_type not in self._kiara.data_type_names:
log_message(
"ignore.operation_config",
operation_type="pretty_print",
module_type=module_cls._module_type_name,
source_type=source_type, # type: ignore
target_type=target_type,
reason=f"Target type '{target_type}' not registered.",
)
continue
func_name = f"pretty_print__{source_type}__as__{target_type}"
attr = getattr(module_cls, func_name)
doc = DocumentationMetadataModel.from_function(attr)
mc = {"source_type": source_type, "target_type": target_type}
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
op_id = self._calculate_op_id(
source_type=source_type, target_type=target_type
)
result[op_id] = oc
for data_type_name, data_type_class in self._kiara.data_type_classes.items():
for attr in data_type_class.__dict__.keys():
if not attr.startswith("pretty_print_as__"):
continue
target_type = attr[17:]
if target_type not in self._kiara.data_type_names:
log_message(
"operation_config.ignore",
operation_type="pretty_print",
source_type=data_type_name,
target_type=target_type,
reason=f"Target type '{target_type}' not registered.",
) # type: ignore
# TODO: inspect signature?
doc = DocumentationMetadataModel.from_string(
f"Pretty print a {data_type_name} value as a {target_type}."
)
mc = {
"source_type": data_type_name,
"target_type": target_type,
}
oc = ManifestOperationConfig(
module_type="pretty_print.value", module_config=mc, doc=doc
)
result[f"_type_{data_type_name}__{target_type}"] = oc
return result.values()
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[PrettyPrintDetails, None]:
details = self.extract_details(module)
if details is None:
return None
else:
return details
def extract_details(self, module: "KiaraModule") -> Union[PrettyPrintDetails, None]:
if len(module.inputs_schema) != 2 or len(module.outputs_schema) != 1:
return None
if "rendered_value" not in module.outputs_schema.keys():
return None
target_type = module.outputs_schema["rendered_value"].type
if "value" not in module.inputs_schema.keys():
return None
if "render_config" not in module.inputs_schema.keys():
return None
input_field_match = "value"
input_field_type = module.inputs_schema[input_field_match].type
operation_id = self._calculate_op_id(
source_type=input_field_type, target_type=target_type
)
details = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": operation_id,
"source_type": input_field_type,
"target_type": target_type,
"is_internal_operation": True,
}
result = PrettyPrintDetails.create_operation_details(**details)
return result
def get_target_types_for(self, source_type: str) -> Mapping[str, Operation]:
# TODO: support for sub-types
result: Dict[str, Operation] = {}
for operation in self.operations.values():
details = self.retrieve_operation_details(operation)
if details.source_type == source_type:
target_type = details.target_type
if target_type in result.keys():
raise Exception(
f"More than one operation for pretty_print combination '{source_type}'/'{target_type}', this is not supported (for now)."
)
result[target_type] = operation
return result
def get_operation_for_render_combination(
self, source_type: str, target_type: str
) -> Operation:
type_lineage = self._kiara.type_registry.get_type_lineage(
data_type_name=source_type
)
for st in type_lineage:
target_types = self.get_target_types_for(source_type=st)
if not target_types:
continue
if target_type not in target_types.keys():
raise Exception(
f"No operation that produces '{target_type}' for source type: {st}."
)
result = target_types[target_type]
return result
raise Exception(f"No pretty_print opration(s) for source type: {source_type}.")
check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/included_core_operations/pretty_print.py
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[PrettyPrintDetails, None]:
details = self.extract_details(module)
if details is None:
return None
else:
return details
extract_details(self, module)
¶Source code in kiara/operations/included_core_operations/pretty_print.py
def extract_details(self, module: "KiaraModule") -> Union[PrettyPrintDetails, None]:
if len(module.inputs_schema) != 2 or len(module.outputs_schema) != 1:
return None
if "rendered_value" not in module.outputs_schema.keys():
return None
target_type = module.outputs_schema["rendered_value"].type
if "value" not in module.inputs_schema.keys():
return None
if "render_config" not in module.inputs_schema.keys():
return None
input_field_match = "value"
input_field_type = module.inputs_schema[input_field_match].type
operation_id = self._calculate_op_id(
source_type=input_field_type, target_type=target_type
)
details = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": operation_id,
"source_type": input_field_type,
"target_type": target_type,
"is_internal_operation": True,
}
result = PrettyPrintDetails.create_operation_details(**details)
return result
get_operation_for_render_combination(self, source_type, target_type)
¶Source code in kiara/operations/included_core_operations/pretty_print.py
def get_operation_for_render_combination(
self, source_type: str, target_type: str
) -> Operation:
type_lineage = self._kiara.type_registry.get_type_lineage(
data_type_name=source_type
)
for st in type_lineage:
target_types = self.get_target_types_for(source_type=st)
if not target_types:
continue
if target_type not in target_types.keys():
raise Exception(
f"No operation that produces '{target_type}' for source type: {st}."
)
result = target_types[target_type]
return result
raise Exception(f"No pretty_print opration(s) for source type: {source_type}.")
get_target_types_for(self, source_type)
¶Source code in kiara/operations/included_core_operations/pretty_print.py
def get_target_types_for(self, source_type: str) -> Mapping[str, Operation]:
# TODO: support for sub-types
result: Dict[str, Operation] = {}
for operation in self.operations.values():
details = self.retrieve_operation_details(operation)
if details.source_type == source_type:
target_type = details.target_type
if target_type in result.keys():
raise Exception(
f"More than one operation for pretty_print combination '{source_type}'/'{target_type}', this is not supported (for now)."
)
result[target_type] = operation
return result
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/included_core_operations/pretty_print.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = {}
for name, module_cls in self._kiara.module_type_classes.items():
if not issubclass(module_cls, PrettyPrintModule):
continue
for (
source_type,
target_type,
) in module_cls.retrieve_supported_render_combinations():
if source_type not in self._kiara.data_type_names:
log_message("ignore.operation_config", operation_type="pretty_print", module_type=module_cls._module_type_name, source_type=source_type, target_type=target_type, reason=f"Source type '{source_type}' not registered.") # type: ignore
continue
if target_type not in self._kiara.data_type_names:
log_message(
"ignore.operation_config",
operation_type="pretty_print",
module_type=module_cls._module_type_name,
source_type=source_type, # type: ignore
target_type=target_type,
reason=f"Target type '{target_type}' not registered.",
)
continue
func_name = f"pretty_print__{source_type}__as__{target_type}"
attr = getattr(module_cls, func_name)
doc = DocumentationMetadataModel.from_function(attr)
mc = {"source_type": source_type, "target_type": target_type}
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
op_id = self._calculate_op_id(
source_type=source_type, target_type=target_type
)
result[op_id] = oc
for data_type_name, data_type_class in self._kiara.data_type_classes.items():
for attr in data_type_class.__dict__.keys():
if not attr.startswith("pretty_print_as__"):
continue
target_type = attr[17:]
if target_type not in self._kiara.data_type_names:
log_message(
"operation_config.ignore",
operation_type="pretty_print",
source_type=data_type_name,
target_type=target_type,
reason=f"Target type '{target_type}' not registered.",
) # type: ignore
# TODO: inspect signature?
doc = DocumentationMetadataModel.from_string(
f"Pretty print a {data_type_name} value as a {target_type}."
)
mc = {
"source_type": data_type_name,
"target_type": target_type,
}
oc = ManifestOperationConfig(
module_type="pretty_print.value", module_config=mc, doc=doc
)
result[f"_type_{data_type_name}__{target_type}"] = oc
return result.values()
render_value
¶logger
¶
RenderValueDetails (BaseOperationDetails)
pydantic-model
¶A model that contains information needed to describe an 'extract_metadata' operation.
Source code in kiara/operations/included_core_operations/render_value.py
class RenderValueDetails(BaseOperationDetails):
"""A model that contains information needed to describe an 'extract_metadata' operation."""
source_data_type: str = Field(description="The data type that will be rendered.")
target_data_type: str = Field(description="The rendered data type.")
RenderValueOperationType (OperationType)
¶An operation that renders a value.
Source code in kiara/operations/included_core_operations/render_value.py
class RenderValueOperationType(OperationType[RenderValueDetails]):
"""An operation that renders a value."""
_operation_type_name = "render_value"
def _calculate_op_id(cls, source_type: str, target_type: str):
if source_type == "any":
operation_id = f"render.as.{target_type}"
else:
operation_id = f"render.{source_type}.as.{target_type}"
return operation_id
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = {}
for name, module_cls in self._kiara.module_type_classes.items():
if not issubclass(module_cls, RenderValueModule):
continue
for (
source_type,
target_type,
) in module_cls.retrieve_supported_render_combinations():
if source_type not in self._kiara.data_type_names:
log_message("ignore.operation_config", operation_type="render_value", module_type=module_cls._module_type_name, source_type=source_type, target_type=target_type, reason=f"Source type '{source_type}' not registered.") # type: ignore
continue
if target_type not in self._kiara.data_type_names:
log_message(
"ignore.operation_config",
operation_type="render_value",
module_type=module_cls._module_type_name,
source_type=source_type, # type: ignore
target_type=target_type,
reason=f"Target type '{target_type}' not registered.",
)
continue
func_name = f"render__{source_type}__as__{target_type}"
attr = getattr(module_cls, func_name)
doc = DocumentationMetadataModel.from_function(attr)
mc = {"source_type": source_type, "target_type": target_type}
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
op_id = self._calculate_op_id(
source_type=source_type, target_type=target_type
)
result[op_id] = oc
for data_type_name, data_type_class in self._kiara.data_type_classes.items():
for attr in data_type_class.__dict__.keys():
if not attr.startswith("render_as__"):
continue
target_type = attr[11:]
if target_type not in self._kiara.data_type_names:
log_message(
"operation_config.ignore",
operation_type="render_value",
source_type=data_type_name,
target_type=target_type,
reason=f"Target type '{target_type}' not registered.",
) # type: ignore
# TODO: inspect signature?
doc = DocumentationMetadataModel.from_string(
f"Render a {data_type_name} value as a {target_type}."
)
mc = {
"source_type": data_type_name,
"target_type": target_type,
}
oc = ManifestOperationConfig(
module_type="render.value", module_config=mc, doc=doc
)
result[f"_type_{data_type_name}_{target_type}"] = oc
return result.values()
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[RenderValueDetails, None]:
if len(module.inputs_schema) != 2:
return None
if len(module.outputs_schema) != 1:
return None
if "value" not in module.inputs_schema.keys():
return None
if (
"render_config" not in module.inputs_schema.keys()
or module.inputs_schema["render_config"].type != "dict"
):
return None
if (
"render_value_result" not in module.outputs_schema.keys()
or module.outputs_schema["render_value_result"].type
!= "render_value_result"
):
return None
source_type = module.inputs_schema["value"].type
target_type = module.get_config_value("target_type")
if source_type == "any":
op_id = f"render.as.{target_type}"
else:
op_id = f"render.{source_type}.as.{target_type}"
details = RenderValueDetails.create_operation_details(
module_inputs_schema=module.inputs_schema,
module_outputs_schema=module.outputs_schema,
operation_id=op_id,
source_data_type=source_type,
target_data_type=target_type,
is_internal_operation=True,
)
return details
def get_render_operations_for_source_type(
self, source_type: str
) -> Mapping[str, Operation]:
"""Return all render operations for the specified data type.
Arguments:
source_type: the data type to render
Returns:
a mapping with the target type as key, and the operation as value
"""
lineage = self._kiara.type_registry.get_type_lineage(data_type_name=source_type)
result: Dict[str, Operation] = {}
for data_type in lineage:
for op_id, op in self.operations.items():
op_details = self.retrieve_operation_details(op)
match = op_details.source_data_type == data_type
if not match:
continue
target_type = op_details.target_data_type
if target_type in result.keys():
continue
result[target_type] = op
return result
def get_render_operation(
self, source_type: str, target_type: str
) -> Union[Operation, None]:
all_ops = self.get_render_operations_for_source_type(source_type=source_type)
return all_ops.get(target_type, None)
check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/included_core_operations/render_value.py
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[RenderValueDetails, None]:
if len(module.inputs_schema) != 2:
return None
if len(module.outputs_schema) != 1:
return None
if "value" not in module.inputs_schema.keys():
return None
if (
"render_config" not in module.inputs_schema.keys()
or module.inputs_schema["render_config"].type != "dict"
):
return None
if (
"render_value_result" not in module.outputs_schema.keys()
or module.outputs_schema["render_value_result"].type
!= "render_value_result"
):
return None
source_type = module.inputs_schema["value"].type
target_type = module.get_config_value("target_type")
if source_type == "any":
op_id = f"render.as.{target_type}"
else:
op_id = f"render.{source_type}.as.{target_type}"
details = RenderValueDetails.create_operation_details(
module_inputs_schema=module.inputs_schema,
module_outputs_schema=module.outputs_schema,
operation_id=op_id,
source_data_type=source_type,
target_data_type=target_type,
is_internal_operation=True,
)
return details
get_render_operation(self, source_type, target_type)
¶Source code in kiara/operations/included_core_operations/render_value.py
def get_render_operation(
self, source_type: str, target_type: str
) -> Union[Operation, None]:
all_ops = self.get_render_operations_for_source_type(source_type=source_type)
return all_ops.get(target_type, None)
get_render_operations_for_source_type(self, source_type)
¶Return all render operations for the specified data type.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
source_type |
str |
the data type to render |
required |
Returns:
| Type | Description |
|---|---|
Mapping[str, kiara.models.module.operation.Operation] |
a mapping with the target type as key, and the operation as value |
Source code in kiara/operations/included_core_operations/render_value.py
def get_render_operations_for_source_type(
self, source_type: str
) -> Mapping[str, Operation]:
"""Return all render operations for the specified data type.
Arguments:
source_type: the data type to render
Returns:
a mapping with the target type as key, and the operation as value
"""
lineage = self._kiara.type_registry.get_type_lineage(data_type_name=source_type)
result: Dict[str, Operation] = {}
for data_type in lineage:
for op_id, op in self.operations.items():
op_details = self.retrieve_operation_details(op)
match = op_details.source_data_type == data_type
if not match:
continue
target_type = op_details.target_data_type
if target_type in result.keys():
continue
result[target_type] = op
return result
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/included_core_operations/render_value.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = {}
for name, module_cls in self._kiara.module_type_classes.items():
if not issubclass(module_cls, RenderValueModule):
continue
for (
source_type,
target_type,
) in module_cls.retrieve_supported_render_combinations():
if source_type not in self._kiara.data_type_names:
log_message("ignore.operation_config", operation_type="render_value", module_type=module_cls._module_type_name, source_type=source_type, target_type=target_type, reason=f"Source type '{source_type}' not registered.") # type: ignore
continue
if target_type not in self._kiara.data_type_names:
log_message(
"ignore.operation_config",
operation_type="render_value",
module_type=module_cls._module_type_name,
source_type=source_type, # type: ignore
target_type=target_type,
reason=f"Target type '{target_type}' not registered.",
)
continue
func_name = f"render__{source_type}__as__{target_type}"
attr = getattr(module_cls, func_name)
doc = DocumentationMetadataModel.from_function(attr)
mc = {"source_type": source_type, "target_type": target_type}
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
op_id = self._calculate_op_id(
source_type=source_type, target_type=target_type
)
result[op_id] = oc
for data_type_name, data_type_class in self._kiara.data_type_classes.items():
for attr in data_type_class.__dict__.keys():
if not attr.startswith("render_as__"):
continue
target_type = attr[11:]
if target_type not in self._kiara.data_type_names:
log_message(
"operation_config.ignore",
operation_type="render_value",
source_type=data_type_name,
target_type=target_type,
reason=f"Target type '{target_type}' not registered.",
) # type: ignore
# TODO: inspect signature?
doc = DocumentationMetadataModel.from_string(
f"Render a {data_type_name} value as a {target_type}."
)
mc = {
"source_type": data_type_name,
"target_type": target_type,
}
oc = ManifestOperationConfig(
module_type="render.value", module_config=mc, doc=doc
)
result[f"_type_{data_type_name}_{target_type}"] = oc
return result.values()
serialize
¶
DeSerializeDetails (BaseOperationDetails)
pydantic-model
¶Source code in kiara/operations/included_core_operations/serialize.py
class DeSerializeDetails(BaseOperationDetails):
value_type: str = Field(
"The name of the input field for the serialized version of the value."
)
value_input_field: str = Field(
"The name of the input field for the serialized version of the value."
)
object_output_field: str = Field(
description="The (output) field name containing the deserialized python class."
)
serialization_profile: str = Field(
description="The name for the serialization profile used on the source value."
)
target_profile: str = Field(description="The target profile name.")
# target_class: PythonClass = Field(
# description="The python class of the result object."
# )
# def retrieve_inputs_schema(self) -> ValueSetSchema:
#
# return {"value": {"type": self.value_type, "doc": "The value to de-serialize."}}
#
# def retrieve_outputs_schema(self) -> ValueSetSchema:
#
# return {
# "python_object": {
# "type": "python_object",
# "doc": "The de-serialized python object instance.",
# }
# }
#
# def create_module_inputs(self, inputs: Mapping[str, Any]) -> Mapping[str, Any]:
#
# result = {self.value_input_field: inputs["value"]}
# return result
#
# def create_operation_outputs(self, outputs: ValueMap) -> Mapping[str, Value]:
#
# return outputs
object_output_field: str
pydantic-field
required
¶The (output) field name containing the deserialized python class.
serialization_profile: str
pydantic-field
required
¶The name for the serialization profile used on the source value.
target_profile: str
pydantic-field
required
¶The target profile name.
value_input_field: str
pydantic-field
¶value_type: str
pydantic-field
¶
DeSerializeOperationType (OperationType)
¶An operation that takes a value, and serializes it into the format suitable to the [serialized_value][kiara.data_types.included_core_types.SeriailzedValue] value type.
For a module profile to be picked up by this operation type, it needs to have:
- exactly one output field of type serialized_value
- either one of (in this order):
- exactly one input field
- one input field where the field name equals the type name
- an input field called 'value'
Source code in kiara/operations/included_core_operations/serialize.py
class DeSerializeOperationType(OperationType[DeSerializeDetails]):
"""An operation that takes a value, and serializes it into the format suitable to the [`serialized_value`][kiara.data_types.included_core_types.SeriailzedValue] value type.
For a module profile to be picked up by this operation type, it needs to have:
- exactly one output field of type `serialized_value`
- either one of (in this order):
- exactly one input field
- one input field where the field name equals the type name
- an input field called 'value'
"""
_operation_type_name = "deserialize"
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = []
for name, module_cls in self._kiara.module_type_classes.items():
if not hasattr(module_cls, "retrieve_serialized_value_type"):
continue
if not hasattr(module_cls, "retrieve_supported_target_profiles"):
continue
if not hasattr(module_cls, "retrieve_supported_serialization_profile"):
continue
try:
value_type = module_cls.retrieve_serialized_value_type() # type: ignore
except TypeError:
raise Exception(
f"Can't retrieve source value type for deserialization module '{module_cls.__name__}'. This is most likely a bug, maybe you are missing a '@classmethod' annotation on the 'retrieve_source_value_type' method?"
)
try:
serialization_profile = module_cls.retrieve_supported_serialization_profile() # type: ignore
except TypeError:
raise Exception(
f"Can't retrieve supported serialization profiles for deserialization module '{module_cls.__name__}'. This is most likely a bug, maybe you are missing a '@classmethod' annotation on the 'retrieve_supported_serialization_profile' method?"
)
try:
target_profiles = module_cls.retrieve_supported_target_profiles() # type: ignore
except TypeError:
raise Exception(
f"Can't retrieve supported target profile for deserialization module '{module_cls.__name__}'. This is most likely a bug, maybe you are missing a '@classmethod' annotation on the 'retrieve_supported_target_profile' method?"
)
for _profile_name, cls in target_profiles.items():
func_name = f"to__{_profile_name}"
attr = getattr(module_cls, func_name)
doc = DocumentationMetadataModel.from_function(attr)
mc = {
"value_type": value_type,
"target_profile": _profile_name,
"serialization_profile": serialization_profile
# "target_class": PythonClass.from_class(cls),
}
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
result.append(oc)
return result
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[DeSerializeDetails, None]:
details = self.extract_details(module)
if details is None:
return None
else:
return details
def extract_details(self, module: "KiaraModule") -> Union[DeSerializeDetails, None]:
result_field_name = None
for field_name, schema in module.outputs_schema.items():
if schema.type != "python_object":
continue
else:
if result_field_name is not None:
log_message(
"ignore.operation",
reason=f"found more than one potential result value field: {result_field_name} -- {field_name}'",
module_type=module.module_type_name,
)
continue
else:
result_field_name = field_name
if not result_field_name:
return None
input_field_name = None
for field_name, schema in module.inputs_schema.items():
if field_name != schema.type:
continue
if input_field_name is not None:
log_message(
"ignore.operation",
reason=f"found more than one potential result value field: {result_field_name} -- {field_name}'",
module_type=module.module_type_name,
)
continue
else:
input_field_name = field_name
if not input_field_name:
return None
try:
value_type = module.config.get("value_type")
target_profile = module.config.get("target_profile")
serialization_profile = module.config.get("serialization_profile")
# target_class = module.config.get("target_class")
except Exception as e:
log_message(
"ignore.operation",
reason=str(e),
module_type=module.module_type_name,
)
return None
if value_type not in self._kiara.type_registry.data_type_names:
log_message(
"ignore.operation",
reason=f"Invalid value type: {value_type}",
module_type=module.module_type_name,
)
return None
if input_field_name == "any":
operation_id = "deserialize.value"
else:
operation_id = f"deserialize.{input_field_name}.as.{target_profile}"
details: Dict[str, Any] = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": operation_id,
"value_type": input_field_name,
"value_input_field": input_field_name,
"object_output_field": result_field_name,
"target_profile": target_profile,
"serialization_profile": serialization_profile,
# "target_class": target_class,
"is_internal_operation": True,
}
result = DeSerializeDetails.construct(**details)
return result
def find_deserialization_operations_for_type(
self, type_name: str
) -> List[Operation]:
lineage = self._kiara.type_registry.get_type_lineage(type_name)
result = []
for data_type in lineage:
match = []
for op in self.operations.values():
details = self.retrieve_operation_details(op)
if details.value_type == data_type:
match.append(op)
result.extend(match)
return result
def find_deserialzation_operation_for_type_and_profile(
self, type_name: str, serialization_profile: str
) -> List[Operation]:
lineage = self._kiara.type_registry.get_type_lineage(type_name)
serialize_ops: List[Operation] = []
for data_type in lineage:
match = []
op = None
for op in self.operations.values():
details = self.retrieve_operation_details(op)
if (
details.value_type == data_type
and details.serialization_profile == serialization_profile
):
match.append(op)
if match:
if len(match) > 1:
assert op is not None
raise Exception(
f"Multiple deserialization operations found for data type '{type_name}' and serialization profile '{serialization_profile}'. This is not supported (yet)."
)
serialize_ops.append(match[0])
return serialize_ops
check_matching_operation(self, module)
¶Check whether the provided module is a valid operation for this type.
Source code in kiara/operations/included_core_operations/serialize.py
def check_matching_operation(
self, module: "KiaraModule"
) -> Union[DeSerializeDetails, None]:
details = self.extract_details(module)
if details is None:
return None
else:
return details
extract_details(self, module)
¶Source code in kiara/operations/included_core_operations/serialize.py
def extract_details(self, module: "KiaraModule") -> Union[DeSerializeDetails, None]:
result_field_name = None
for field_name, schema in module.outputs_schema.items():
if schema.type != "python_object":
continue
else:
if result_field_name is not None:
log_message(
"ignore.operation",
reason=f"found more than one potential result value field: {result_field_name} -- {field_name}'",
module_type=module.module_type_name,
)
continue
else:
result_field_name = field_name
if not result_field_name:
return None
input_field_name = None
for field_name, schema in module.inputs_schema.items():
if field_name != schema.type:
continue
if input_field_name is not None:
log_message(
"ignore.operation",
reason=f"found more than one potential result value field: {result_field_name} -- {field_name}'",
module_type=module.module_type_name,
)
continue
else:
input_field_name = field_name
if not input_field_name:
return None
try:
value_type = module.config.get("value_type")
target_profile = module.config.get("target_profile")
serialization_profile = module.config.get("serialization_profile")
# target_class = module.config.get("target_class")
except Exception as e:
log_message(
"ignore.operation",
reason=str(e),
module_type=module.module_type_name,
)
return None
if value_type not in self._kiara.type_registry.data_type_names:
log_message(
"ignore.operation",
reason=f"Invalid value type: {value_type}",
module_type=module.module_type_name,
)
return None
if input_field_name == "any":
operation_id = "deserialize.value"
else:
operation_id = f"deserialize.{input_field_name}.as.{target_profile}"
details: Dict[str, Any] = {
"module_inputs_schema": module.inputs_schema,
"module_outputs_schema": module.outputs_schema,
"operation_id": operation_id,
"value_type": input_field_name,
"value_input_field": input_field_name,
"object_output_field": result_field_name,
"target_profile": target_profile,
"serialization_profile": serialization_profile,
# "target_class": target_class,
"is_internal_operation": True,
}
result = DeSerializeDetails.construct(**details)
return result
find_deserialization_operations_for_type(self, type_name)
¶Source code in kiara/operations/included_core_operations/serialize.py
def find_deserialization_operations_for_type(
self, type_name: str
) -> List[Operation]:
lineage = self._kiara.type_registry.get_type_lineage(type_name)
result = []
for data_type in lineage:
match = []
for op in self.operations.values():
details = self.retrieve_operation_details(op)
if details.value_type == data_type:
match.append(op)
result.extend(match)
return result
find_deserialzation_operation_for_type_and_profile(self, type_name, serialization_profile)
¶Source code in kiara/operations/included_core_operations/serialize.py
def find_deserialzation_operation_for_type_and_profile(
self, type_name: str, serialization_profile: str
) -> List[Operation]:
lineage = self._kiara.type_registry.get_type_lineage(type_name)
serialize_ops: List[Operation] = []
for data_type in lineage:
match = []
op = None
for op in self.operations.values():
details = self.retrieve_operation_details(op)
if (
details.value_type == data_type
and details.serialization_profile == serialization_profile
):
match.append(op)
if match:
if len(match) > 1:
assert op is not None
raise Exception(
f"Multiple deserialization operations found for data type '{type_name}' and serialization profile '{serialization_profile}'. This is not supported (yet)."
)
serialize_ops.append(match[0])
return serialize_ops
retrieve_included_operation_configs(self)
¶Source code in kiara/operations/included_core_operations/serialize.py
def retrieve_included_operation_configs(
self,
) -> Iterable[Union[Mapping, OperationConfig]]:
result = []
for name, module_cls in self._kiara.module_type_classes.items():
if not hasattr(module_cls, "retrieve_serialized_value_type"):
continue
if not hasattr(module_cls, "retrieve_supported_target_profiles"):
continue
if not hasattr(module_cls, "retrieve_supported_serialization_profile"):
continue
try:
value_type = module_cls.retrieve_serialized_value_type() # type: ignore
except TypeError:
raise Exception(
f"Can't retrieve source value type for deserialization module '{module_cls.__name__}'. This is most likely a bug, maybe you are missing a '@classmethod' annotation on the 'retrieve_source_value_type' method?"
)
try:
serialization_profile = module_cls.retrieve_supported_serialization_profile() # type: ignore
except TypeError:
raise Exception(
f"Can't retrieve supported serialization profiles for deserialization module '{module_cls.__name__}'. This is most likely a bug, maybe you are missing a '@classmethod' annotation on the 'retrieve_supported_serialization_profile' method?"
)
try:
target_profiles = module_cls.retrieve_supported_target_profiles() # type: ignore
except TypeError:
raise Exception(
f"Can't retrieve supported target profile for deserialization module '{module_cls.__name__}'. This is most likely a bug, maybe you are missing a '@classmethod' annotation on the 'retrieve_supported_target_profile' method?"
)
for _profile_name, cls in target_profiles.items():
func_name = f"to__{_profile_name}"
attr = getattr(module_cls, func_name)
doc = DocumentationMetadataModel.from_function(attr)
mc = {
"value_type": value_type,
"target_profile": _profile_name,
"serialization_profile": serialization_profile
# "target_class": PythonClass.from_class(cls),
}
oc = ManifestOperationConfig(
module_type=name, module_config=mc, doc=doc
)
result.append(oc)
return result
processing
special
¶
log
¶
Classes¶
JobStatusListener (Protocol)
¶
Source code in kiara/processing/__init__.py
class JobStatusListener(Protocol):
def job_status_changed(
self,
job_id: uuid.UUID,
old_status: Union[JobStatus, None],
new_status: JobStatus,
):
pass
job_status_changed(self, job_id, old_status, new_status)
¶Source code in kiara/processing/__init__.py
def job_status_changed(
self,
job_id: uuid.UUID,
old_status: Union[JobStatus, None],
new_status: JobStatus,
):
pass
ModuleProcessor (ABC)
¶
Source code in kiara/processing/__init__.py
class ModuleProcessor(abc.ABC):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._created_jobs: Dict[uuid.UUID, Dict[str, Any]] = {}
self._running_job_details: Dict[uuid.UUID, Dict[str, Any]] = {}
self._active_jobs: Dict[uuid.UUID, ActiveJob] = {}
self._failed_jobs: Dict[uuid.UUID, ActiveJob] = {}
self._finished_jobs: Dict[uuid.UUID, ActiveJob] = {}
self._output_refs: Dict[uuid.UUID, ValueMapWritable] = {}
self._job_records: Dict[uuid.UUID, JobRecord] = {}
self._listeners: List[JobStatusListener] = []
def _send_job_event(
self,
job_id: uuid.UUID,
old_status: Union[JobStatus, None],
new_status: JobStatus,
):
for listener in self._listeners:
listener.job_status_changed(
job_id=job_id, old_status=old_status, new_status=new_status
)
def register_job_status_listener(self, listener: JobStatusListener):
self._listeners.append(listener)
def get_job(self, job_id: uuid.UUID) -> ActiveJob:
if job_id in self._active_jobs.keys():
return self._active_jobs[job_id]
elif job_id in self._finished_jobs.keys():
return self._finished_jobs[job_id]
elif job_id in self._failed_jobs.keys():
return self._failed_jobs[job_id]
else:
raise Exception(f"No job with id '{job_id}' registered.")
def get_job_status(self, job_id: uuid.UUID) -> JobStatus:
job = self.get_job(job_id=job_id)
return job.status
def get_job_record(self, job_id: uuid.UUID) -> JobRecord:
if job_id in self._job_records.keys():
return self._job_records[job_id]
else:
raise Exception(f"No job record for job with id '{job_id}' registered.")
def create_job(
self, job_config: JobConfig, job_metadata: Union[None, Mapping[str, Any]]
) -> uuid.UUID:
environments = {
env_name: env.instance_id
for env_name, env in self._kiara.current_environments.items()
}
if job_metadata is None:
job_metadata = {}
result_pedigree = ValuePedigree(
kiara_id=self._kiara.id,
module_type=job_config.module_type,
module_config=job_config.module_config,
inputs=job_config.inputs,
environments=environments,
)
module = self._kiara.create_module(manifest=job_config)
unique_result_values = module.characteristics.unique_result_values
outputs = ValueMapWritable.create_from_schema(
kiara=self._kiara,
schema=module.outputs_schema,
pedigree=result_pedigree,
unique_value_ids=unique_result_values,
)
job_id = ID_REGISTRY.generate(kiara_id=self._kiara.id)
job_log = JobLog()
job = ActiveJob.construct(job_id=job_id, job_config=job_config, job_log=job_log)
ID_REGISTRY.update_metadata(job_id, obj=job)
job.job_log.add_log("job created")
job_details = {
"job_config": job_config,
"job": job,
"module": module,
"outputs": outputs,
"job_metadata": job_metadata,
}
self._created_jobs[job_id] = job_details
self._send_job_event(
job_id=job_id, old_status=None, new_status=JobStatus.CREATED
)
if is_develop():
dev_settings = get_dev_config()
if dev_settings.log.log_pre_run and (
not module.characteristics.is_internal
or dev_settings.log.pre_run.internal_modules
):
is_pipeline_step = job_metadata.get("is_pipeline_step", False)
if is_pipeline_step:
if dev_settings.log.pre_run.pipeline_steps:
step_id = job_metadata.get("step_id", None)
assert step_id is not None
title = (
f"Pre-run information for pipeline step: [i]{step_id}[/i]"
)
else:
title = None
else:
title = f"Pre-run information for module: [i]{module.module_type_name}[/i]"
if title:
from kiara.utils.debug import create_module_preparation_table
from kiara.utils.develop import log_dev_message
table = create_module_preparation_table(
kiara=self._kiara,
job_config=job_config,
job_id=job_id,
module=module,
)
log_dev_message(table, title=title)
return job_id
def queue_job(self, job_id: uuid.UUID) -> ActiveJob:
job_details = self._created_jobs.pop(job_id)
self._running_job_details[job_id] = job_details
job_config: JobConfig = job_details.get("job_config") # type: ignore
job: ActiveJob = job_details.get("job") # type: ignore
module: KiaraModule = job_details.get("module") # type: ignore
outputs: ValueMapWritable = job_details.get("outputs") # type: ignore
self._active_jobs[job_id] = job # type: ignore
self._output_refs[job_id] = outputs # type: ignore
input_values = self._kiara.data_registry.load_values(job_config.inputs)
if module.is_pipeline():
module._set_job_registry(self._kiara.job_registry) # type: ignore
try:
self._add_processing_task(
job_id=job_id,
module=module,
inputs=input_values,
outputs=outputs,
job_log=job.job_log,
)
return job
except Exception as e:
msg = str(e)
if not msg:
msg = repr(e)
job.error = msg
if isinstance(e, KiaraProcessingException):
e._module = module
e._inputs = ValueMapReadOnly.create_from_ids(
data_registry=self._kiara.data_registry, **job_config.inputs
)
job._exception = e
log_exception(e)
raise e
else:
kpe = KiaraProcessingException(
e,
module=module,
inputs=ValueMapReadOnly.create_from_ids(
self._kiara.data_registry, **job_config.inputs
),
)
job._exception = kpe
log_exception(kpe)
raise e
def job_status_updated(
self, job_id: uuid.UUID, status: Union[JobStatus, str, Exception]
):
job = self._active_jobs.get(job_id, None)
if job is None:
raise Exception(
f"Can't retrieve active job with id '{job_id}', no such job registered."
)
old_status = job.status
if status == JobStatus.SUCCESS:
self._active_jobs.pop(job_id)
job.job_log.add_log("job finished successfully")
job.status = JobStatus.SUCCESS
job.finished = datetime.now()
values = self._output_refs[job_id]
values.sync_values()
value_ids = values.get_all_value_ids()
job.results = value_ids
job.job_log.percent_finished = 100
job_record = JobRecord.from_active_job(active_job=job, kiara=self._kiara)
self._job_records[job_id] = job_record
self._finished_jobs[job_id] = job
elif status == JobStatus.FAILED or isinstance(status, (str, Exception)):
self._active_jobs.pop(job_id)
job.job_log.add_log("job failed")
job.status = JobStatus.FAILED
job.finished = datetime.now()
if isinstance(status, str):
job.error = status
elif isinstance(status, Exception):
msg = str(status)
job.error = msg
job._exception = status
self._failed_jobs[job_id] = job
log.debug(
"job.failed",
job_id=str(job.job_id),
msg=job.error,
module_type=job.job_config.module_type,
)
elif status == JobStatus.STARTED:
job.job_log.add_log("job started")
job.status = JobStatus.STARTED
job.started = datetime.now()
else:
raise ValueError(f"Invalid value for status: {status}")
log.debug(
"job.status_updated",
old_status=old_status.value,
new_status=job.status.value,
job_id=str(job.job_id),
module_type=job.job_config.module_type,
)
if status in [JobStatus.SUCCESS, JobStatus.FAILED]:
if is_develop():
dev_config = get_dev_config()
if dev_config.log.log_post_run:
details = self._running_job_details[job_id]
module: KiaraModule = details["module"]
skip = False
if (
module.characteristics.is_internal
and not dev_config.log.post_run.internal_modules
):
skip = True
is_pipeline_step = details["job_metadata"].get(
"is_pipeline_step", False
)
if is_pipeline_step and not dev_config.log.post_run.pipeline_steps:
skip = True
if not skip:
if is_pipeline_step:
step_id = details["job_metadata"]["step_id"]
title = f"Post-run information for pipeline step: {step_id}"
else:
title = f"Post-run information for module: {module.module_type_name}"
from kiara.utils.debug import create_post_run_table
from kiara.utils.develop import log_dev_message
rendered = create_post_run_table(
kiara=self._kiara,
job=job,
module=module,
job_config=details["job_config"],
)
log_dev_message(rendered, title=title)
self._running_job_details.pop(job_id)
self._send_job_event(
job_id=job_id, old_status=old_status, new_status=job.status
)
def wait_for(self, *job_ids: uuid.UUID):
"""Wait for the jobs with the specified ids, also optionally sync their outputs with the pipeline value state."""
self._wait_for(*job_ids)
for job_id in job_ids:
job = self._job_records.get(job_id, None)
if job is None:
_job = self._failed_jobs.get(job_id, None)
if _job is None:
raise Exception(f"Can't find job with id: {job_id}")
@abc.abstractmethod
def _wait_for(self, *job_ids: uuid.UUID):
pass
@abc.abstractmethod
def _add_processing_task(
self,
job_id: uuid.UUID,
module: "KiaraModule",
inputs: ValueMap,
outputs: ValueMapWritable,
job_log: JobLog,
) -> str:
pass
Methods¶
create_job(self, job_config, job_metadata)
¶Source code in kiara/processing/__init__.py
def create_job(
self, job_config: JobConfig, job_metadata: Union[None, Mapping[str, Any]]
) -> uuid.UUID:
environments = {
env_name: env.instance_id
for env_name, env in self._kiara.current_environments.items()
}
if job_metadata is None:
job_metadata = {}
result_pedigree = ValuePedigree(
kiara_id=self._kiara.id,
module_type=job_config.module_type,
module_config=job_config.module_config,
inputs=job_config.inputs,
environments=environments,
)
module = self._kiara.create_module(manifest=job_config)
unique_result_values = module.characteristics.unique_result_values
outputs = ValueMapWritable.create_from_schema(
kiara=self._kiara,
schema=module.outputs_schema,
pedigree=result_pedigree,
unique_value_ids=unique_result_values,
)
job_id = ID_REGISTRY.generate(kiara_id=self._kiara.id)
job_log = JobLog()
job = ActiveJob.construct(job_id=job_id, job_config=job_config, job_log=job_log)
ID_REGISTRY.update_metadata(job_id, obj=job)
job.job_log.add_log("job created")
job_details = {
"job_config": job_config,
"job": job,
"module": module,
"outputs": outputs,
"job_metadata": job_metadata,
}
self._created_jobs[job_id] = job_details
self._send_job_event(
job_id=job_id, old_status=None, new_status=JobStatus.CREATED
)
if is_develop():
dev_settings = get_dev_config()
if dev_settings.log.log_pre_run and (
not module.characteristics.is_internal
or dev_settings.log.pre_run.internal_modules
):
is_pipeline_step = job_metadata.get("is_pipeline_step", False)
if is_pipeline_step:
if dev_settings.log.pre_run.pipeline_steps:
step_id = job_metadata.get("step_id", None)
assert step_id is not None
title = (
f"Pre-run information for pipeline step: [i]{step_id}[/i]"
)
else:
title = None
else:
title = f"Pre-run information for module: [i]{module.module_type_name}[/i]"
if title:
from kiara.utils.debug import create_module_preparation_table
from kiara.utils.develop import log_dev_message
table = create_module_preparation_table(
kiara=self._kiara,
job_config=job_config,
job_id=job_id,
module=module,
)
log_dev_message(table, title=title)
return job_id
get_job(self, job_id)
¶Source code in kiara/processing/__init__.py
def get_job(self, job_id: uuid.UUID) -> ActiveJob:
if job_id in self._active_jobs.keys():
return self._active_jobs[job_id]
elif job_id in self._finished_jobs.keys():
return self._finished_jobs[job_id]
elif job_id in self._failed_jobs.keys():
return self._failed_jobs[job_id]
else:
raise Exception(f"No job with id '{job_id}' registered.")
get_job_record(self, job_id)
¶Source code in kiara/processing/__init__.py
def get_job_record(self, job_id: uuid.UUID) -> JobRecord:
if job_id in self._job_records.keys():
return self._job_records[job_id]
else:
raise Exception(f"No job record for job with id '{job_id}' registered.")
get_job_status(self, job_id)
¶Source code in kiara/processing/__init__.py
def get_job_status(self, job_id: uuid.UUID) -> JobStatus:
job = self.get_job(job_id=job_id)
return job.status
job_status_updated(self, job_id, status)
¶Source code in kiara/processing/__init__.py
def job_status_updated(
self, job_id: uuid.UUID, status: Union[JobStatus, str, Exception]
):
job = self._active_jobs.get(job_id, None)
if job is None:
raise Exception(
f"Can't retrieve active job with id '{job_id}', no such job registered."
)
old_status = job.status
if status == JobStatus.SUCCESS:
self._active_jobs.pop(job_id)
job.job_log.add_log("job finished successfully")
job.status = JobStatus.SUCCESS
job.finished = datetime.now()
values = self._output_refs[job_id]
values.sync_values()
value_ids = values.get_all_value_ids()
job.results = value_ids
job.job_log.percent_finished = 100
job_record = JobRecord.from_active_job(active_job=job, kiara=self._kiara)
self._job_records[job_id] = job_record
self._finished_jobs[job_id] = job
elif status == JobStatus.FAILED or isinstance(status, (str, Exception)):
self._active_jobs.pop(job_id)
job.job_log.add_log("job failed")
job.status = JobStatus.FAILED
job.finished = datetime.now()
if isinstance(status, str):
job.error = status
elif isinstance(status, Exception):
msg = str(status)
job.error = msg
job._exception = status
self._failed_jobs[job_id] = job
log.debug(
"job.failed",
job_id=str(job.job_id),
msg=job.error,
module_type=job.job_config.module_type,
)
elif status == JobStatus.STARTED:
job.job_log.add_log("job started")
job.status = JobStatus.STARTED
job.started = datetime.now()
else:
raise ValueError(f"Invalid value for status: {status}")
log.debug(
"job.status_updated",
old_status=old_status.value,
new_status=job.status.value,
job_id=str(job.job_id),
module_type=job.job_config.module_type,
)
if status in [JobStatus.SUCCESS, JobStatus.FAILED]:
if is_develop():
dev_config = get_dev_config()
if dev_config.log.log_post_run:
details = self._running_job_details[job_id]
module: KiaraModule = details["module"]
skip = False
if (
module.characteristics.is_internal
and not dev_config.log.post_run.internal_modules
):
skip = True
is_pipeline_step = details["job_metadata"].get(
"is_pipeline_step", False
)
if is_pipeline_step and not dev_config.log.post_run.pipeline_steps:
skip = True
if not skip:
if is_pipeline_step:
step_id = details["job_metadata"]["step_id"]
title = f"Post-run information for pipeline step: {step_id}"
else:
title = f"Post-run information for module: {module.module_type_name}"
from kiara.utils.debug import create_post_run_table
from kiara.utils.develop import log_dev_message
rendered = create_post_run_table(
kiara=self._kiara,
job=job,
module=module,
job_config=details["job_config"],
)
log_dev_message(rendered, title=title)
self._running_job_details.pop(job_id)
self._send_job_event(
job_id=job_id, old_status=old_status, new_status=job.status
)
queue_job(self, job_id)
¶Source code in kiara/processing/__init__.py
def queue_job(self, job_id: uuid.UUID) -> ActiveJob:
job_details = self._created_jobs.pop(job_id)
self._running_job_details[job_id] = job_details
job_config: JobConfig = job_details.get("job_config") # type: ignore
job: ActiveJob = job_details.get("job") # type: ignore
module: KiaraModule = job_details.get("module") # type: ignore
outputs: ValueMapWritable = job_details.get("outputs") # type: ignore
self._active_jobs[job_id] = job # type: ignore
self._output_refs[job_id] = outputs # type: ignore
input_values = self._kiara.data_registry.load_values(job_config.inputs)
if module.is_pipeline():
module._set_job_registry(self._kiara.job_registry) # type: ignore
try:
self._add_processing_task(
job_id=job_id,
module=module,
inputs=input_values,
outputs=outputs,
job_log=job.job_log,
)
return job
except Exception as e:
msg = str(e)
if not msg:
msg = repr(e)
job.error = msg
if isinstance(e, KiaraProcessingException):
e._module = module
e._inputs = ValueMapReadOnly.create_from_ids(
data_registry=self._kiara.data_registry, **job_config.inputs
)
job._exception = e
log_exception(e)
raise e
else:
kpe = KiaraProcessingException(
e,
module=module,
inputs=ValueMapReadOnly.create_from_ids(
self._kiara.data_registry, **job_config.inputs
),
)
job._exception = kpe
log_exception(kpe)
raise e
register_job_status_listener(self, listener)
¶Source code in kiara/processing/__init__.py
def register_job_status_listener(self, listener: JobStatusListener):
self._listeners.append(listener)
wait_for(self, *job_ids)
¶Wait for the jobs with the specified ids, also optionally sync their outputs with the pipeline value state.
Source code in kiara/processing/__init__.py
def wait_for(self, *job_ids: uuid.UUID):
"""Wait for the jobs with the specified ids, also optionally sync their outputs with the pipeline value state."""
self._wait_for(*job_ids)
for job_id in job_ids:
job = self._job_records.get(job_id, None)
if job is None:
_job = self._failed_jobs.get(job_id, None)
if _job is None:
raise Exception(f"Can't find job with id: {job_id}")
ProcessorConfig (BaseModel)
pydantic-model
¶
Source code in kiara/processing/__init__.py
class ProcessorConfig(BaseModel):
module_processor_type: Literal["synchronous", "multi-threaded"] = "synchronous"
module_processor_type: Literal['synchronous', 'multi-threaded']
pydantic-field
¶
synchronous
¶
SynchronousProcessor (ModuleProcessor)
¶Source code in kiara/processing/synchronous.py
class SynchronousProcessor(ModuleProcessor):
def _add_processing_task(
self,
job_id: uuid.UUID,
module: "KiaraModule",
inputs: ValueMap,
outputs: ValueMapWritable,
job_log: JobLog,
):
self.job_status_updated(job_id=job_id, status=JobStatus.STARTED)
try:
module.process_step(inputs=inputs, outputs=outputs, job_log=job_log)
# output_wrap._sync()
self.job_status_updated(job_id=job_id, status=JobStatus.SUCCESS)
except Exception as e:
self.job_status_updated(job_id=job_id, status=e)
def _wait_for(self, *job_ids: uuid.UUID):
# jobs will always be finished, since we were waiting for them in the 'process' method
return
SynchronousProcessorConfig (ProcessorConfig)
pydantic-model
¶Source code in kiara/processing/synchronous.py
class SynchronousProcessorConfig(ProcessorConfig):
pass
registries
special
¶
ARCHIVE_CONFIG_CLS
¶
NON_ARCHIVE_DETAILS
¶
logger
¶
Classes¶
ArchiveConfig (BaseModel)
pydantic-model
¶
Source code in kiara/registries/__init__.py
class ArchiveConfig(BaseModel):
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
Config
¶Source code in kiara/registries/__init__.py
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
json_loads
¶json_dumps(v, *, default=None, **args)
¶Source code in kiara/registries/__init__.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
ArchiveDetails (BaseModel)
pydantic-model
¶
BaseArchive (KiaraArchive, Generic)
¶
Source code in kiara/registries/__init__.py
class BaseArchive(KiaraArchive, Generic[ARCHIVE_CONFIG_CLS]):
_config_cls = ArchiveConfig
def __init__(self, archive_id: uuid.UUID, config: ARCHIVE_CONFIG_CLS):
self._archive_id: uuid.UUID = archive_id
self._config: ARCHIVE_CONFIG_CLS = config
self._kiara: Union["Kiara", None] = None
def _get_config(self) -> ARCHIVE_CONFIG_CLS:
return self._config
def retrieve_archive_id(self) -> uuid.UUID:
return self._archive_id
@property
def kiara_context(self) -> "Kiara":
if self._kiara is None:
raise Exception("Archive not registered into a kiara context yet.")
return self._kiara
def register_archive(self, kiara: "Kiara"):
if self._kiara is not None:
raise Exception("Archive already registered in a context.")
self._kiara = kiara
def _delete_archive(self):
logger.info(
"ignore.archive_delete_request",
reason="not implemented/applicable",
archive_id=self.archive_id,
item_types=self.supported_item_types(),
archive_type=self.__class__.__name__,
)
kiara_context: Kiara
property
readonly
¶
_config_cls (BaseModel)
private
pydantic-model
¶Source code in kiara/registries/__init__.py
class ArchiveConfig(BaseModel):
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
Config
¶Source code in kiara/registries/__init__.py
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
json_loads
¶json_dumps(v, *, default=None, **args)
¶Source code in kiara/registries/__init__.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
register_archive(self, kiara)
¶Source code in kiara/registries/__init__.py
def register_archive(self, kiara: "Kiara"):
if self._kiara is not None:
raise Exception("Archive already registered in a context.")
self._kiara = kiara
retrieve_archive_id(self)
¶Source code in kiara/registries/__init__.py
def retrieve_archive_id(self) -> uuid.UUID:
return self._archive_id
FileSystemArchiveConfig (ArchiveConfig)
pydantic-model
¶
KiaraArchive (ABC)
¶
Source code in kiara/registries/__init__.py
class KiaraArchive(abc.ABC):
_config_cls: ClassVar[Type[ArchiveConfig]] = ArchiveConfig
@classmethod
@abc.abstractmethod
def supported_item_types(cls) -> Iterable[str]:
pass
@classmethod
@abc.abstractmethod
def is_writeable(cls) -> bool:
pass
@abc.abstractmethod
def register_archive(self, kiara: "Kiara"):
pass
@abc.abstractmethod
def retrieve_archive_id(self) -> uuid.UUID:
pass
@property
def archive_id(self) -> uuid.UUID:
return self.retrieve_archive_id()
@property
def config(self) -> ArchiveConfig:
return self._get_config()
@abc.abstractmethod
def _get_config(self) -> ArchiveConfig:
pass
def get_archive_details(self) -> ArchiveDetails:
return NON_ARCHIVE_DETAILS
def delete_archive(self, archive_id: Union[uuid.UUID, None] = None):
if archive_id != self.archive_id:
raise Exception(
f"Not deleting archive with id '{self.archive_id}': confirmation id '{archive_id}' does not match."
)
logger.info(
"deleteing.archive",
archive_id=self.archive_id,
item_types=self.supported_item_types(),
archive_type=self.__class__.__name__,
)
self._delete_archive()
@abc.abstractmethod
def _delete_archive(self):
pass
def __hash__(self):
return hash(self.archive_id)
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
return self.archive_id == other.archive_id
archive_id: UUID
property
readonly
¶config: ArchiveConfig
property
readonly
¶
_config_cls (BaseModel)
private
pydantic-model
¶Source code in kiara/registries/__init__.py
class ArchiveConfig(BaseModel):
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
Config
¶Source code in kiara/registries/__init__.py
class Config:
json_loads = orjson.loads
json_dumps = orjson_dumps
json_loads
¶json_dumps(v, *, default=None, **args)
¶Source code in kiara/registries/__init__.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
delete_archive(self, archive_id=None)
¶Source code in kiara/registries/__init__.py
def delete_archive(self, archive_id: Union[uuid.UUID, None] = None):
if archive_id != self.archive_id:
raise Exception(
f"Not deleting archive with id '{self.archive_id}': confirmation id '{archive_id}' does not match."
)
logger.info(
"deleteing.archive",
archive_id=self.archive_id,
item_types=self.supported_item_types(),
archive_type=self.__class__.__name__,
)
self._delete_archive()
get_archive_details(self)
¶Source code in kiara/registries/__init__.py
def get_archive_details(self) -> ArchiveDetails:
return NON_ARCHIVE_DETAILS
is_writeable()
classmethod
¶Source code in kiara/registries/__init__.py
@classmethod
@abc.abstractmethod
def is_writeable(cls) -> bool:
pass
register_archive(self, kiara)
¶Source code in kiara/registries/__init__.py
@abc.abstractmethod
def register_archive(self, kiara: "Kiara"):
pass
retrieve_archive_id(self)
¶Source code in kiara/registries/__init__.py
@abc.abstractmethod
def retrieve_archive_id(self) -> uuid.UUID:
pass
supported_item_types()
classmethod
¶Source code in kiara/registries/__init__.py
@classmethod
@abc.abstractmethod
def supported_item_types(cls) -> Iterable[str]:
pass
Modules¶
aliases
special
¶
logger
¶Classes¶
AliasArchive (BaseArchive)
¶Source code in kiara/registries/aliases/__init__.py
class AliasArchive(BaseArchive):
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["alias"]
@abc.abstractmethod
def retrieve_all_aliases(self) -> Union[Mapping[str, uuid.UUID], None]:
"""Retrieve a list of all aliases registered in this archive.
The result of this method can be 'None', for cases where the aliases are determined dynamically.
In kiara, the result of this method is mostly used to improve performance when looking up an alias.
Returns:
a list of strings (the aliases), or 'None' if this archive does not support alias indexes.
"""
@abc.abstractmethod
def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]:
pass
@abc.abstractmethod
def find_aliases_for_value_id(self, value_id: uuid.UUID) -> Union[Set[str], None]:
pass
@classmethod
def is_writeable(cls) -> bool:
return False
find_aliases_for_value_id(self, value_id)
¶Source code in kiara/registries/aliases/__init__.py
@abc.abstractmethod
def find_aliases_for_value_id(self, value_id: uuid.UUID) -> Union[Set[str], None]:
pass
find_value_id_for_alias(self, alias)
¶Source code in kiara/registries/aliases/__init__.py
@abc.abstractmethod
def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]:
pass
is_writeable()
classmethod
¶Source code in kiara/registries/aliases/__init__.py
@classmethod
def is_writeable(cls) -> bool:
return False
retrieve_all_aliases(self)
¶Retrieve a list of all aliases registered in this archive.
The result of this method can be 'None', for cases where the aliases are determined dynamically. In kiara, the result of this method is mostly used to improve performance when looking up an alias.
Returns:
| Type | Description |
|---|---|
Optional[Mapping[str, uuid.UUID]] |
a list of strings (the aliases), or 'None' if this archive does not support alias indexes. |
Source code in kiara/registries/aliases/__init__.py
@abc.abstractmethod
def retrieve_all_aliases(self) -> Union[Mapping[str, uuid.UUID], None]:
"""Retrieve a list of all aliases registered in this archive.
The result of this method can be 'None', for cases where the aliases are determined dynamically.
In kiara, the result of this method is mostly used to improve performance when looking up an alias.
Returns:
a list of strings (the aliases), or 'None' if this archive does not support alias indexes.
"""
supported_item_types()
classmethod
¶Source code in kiara/registries/aliases/__init__.py
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["alias"]
AliasItem (tuple)
¶AliasItem(full_alias, rel_alias, value_id, alias_archive, alias_archive_id)
Source code in kiara/registries/aliases/__init__.py
class AliasItem(NamedTuple):
full_alias: str
rel_alias: str
value_id: uuid.UUID
alias_archive: str
alias_archive_id: uuid.UUID
AliasRegistry
¶Source code in kiara/registries/aliases/__init__.py
class AliasRegistry(object):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._event_callback: Callable = self._kiara.event_registry.add_producer(self)
self._alias_archives: Dict[str, AliasArchive] = {}
"""All registered archives/stores."""
self._default_alias_store: Union[str, None] = None
"""The alias of the store where new aliases are stored by default."""
self._cached_aliases: Union[Dict[str, AliasItem], None] = None
self._cached_aliases_by_id: Union[Dict[uuid.UUID, Set[AliasItem]], None] = None
def register_archive(
self,
archive: AliasArchive,
alias: str = None,
set_as_default_store: Union[bool, None] = None,
):
alias_archive_id = archive.archive_id
archive.register_archive(kiara=self._kiara)
if alias is None:
alias = str(alias_archive_id)
if "." in alias:
raise Exception(
f"Can't register alias archive with as '{alias}': registered name is not allowed to contain a '.' character (yet)."
)
if alias in self._alias_archives.keys():
raise Exception(f"Can't add store, alias '{alias}' already registered.")
self._alias_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, AliasStore):
is_store = True
if set_as_default_store and self._default_alias_store is not None:
raise Exception(
f"Can't set alias store '{alias}' as default store: default store already set."
)
if self._default_alias_store is None:
is_default_store = True
self._default_alias_store = alias
event = AliasArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
alias_archive_id=archive.archive_id,
alias_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
@property
def default_alias_store(self) -> str:
if self._default_alias_store is None:
raise Exception("No default alias store set (yet).")
return self._default_alias_store
@property
def alias_archives(self) -> Mapping[str, AliasArchive]:
return self._alias_archives
def get_archive(
self, archive_id: Union[str, None] = None
) -> Union[AliasArchive, None]:
if archive_id is None:
archive_id = self.default_alias_store
if archive_id is None:
raise Exception("Can't retrieve default alias archive, none set (yet).")
archive = self._alias_archives.get(archive_id, None)
return archive
@property
def all_aliases(self) -> Iterable[str]:
return self.aliases.keys()
@property
def aliases_by_id(self) -> Mapping[uuid.UUID, Set[AliasItem]]:
if self._cached_aliases_by_id is None:
self.aliases # noqa
return self._cached_aliases_by_id # type: ignore
@property
def aliases(self) -> Dict[str, AliasItem]:
"""Retrieve a map of all available aliases, context wide, with the registered archive aliases as values."""
if self._cached_aliases is not None:
return self._cached_aliases
# TODO: multithreading lock
all_aliases: Dict[str, AliasItem] = {}
all_aliases_by_id: Dict[uuid.UUID, Set[AliasItem]] = {}
for archive_alias, archive in self._alias_archives.items():
alias_map = archive.retrieve_all_aliases()
if alias_map is None:
continue
for alias, v_id in alias_map.items():
if archive_alias == self.default_alias_store:
final_alias = alias
else:
final_alias = f"{archive_alias}.{alias}"
if final_alias in all_aliases.keys():
raise Exception(
f"Inconsistent alias registry: alias '{final_alias}' available more than once."
)
item = AliasItem(
full_alias=final_alias,
rel_alias=alias,
value_id=v_id,
alias_archive=archive_alias,
alias_archive_id=archive.archive_id,
)
all_aliases[final_alias] = item
all_aliases_by_id.setdefault(v_id, set()).add(item)
self._cached_aliases = {k: all_aliases[k] for k in sorted(all_aliases.keys())}
self._cached_aliases_by_id = all_aliases_by_id
return self._cached_aliases
def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]:
alias_item = self.aliases.get(alias, None)
if alias_item is not None:
return alias_item.value_id
if "." not in alias:
return None
archive_id, rest = alias.split(".", maxsplit=2)
archive = self.get_archive(archive_id=archive_id)
if archive is None:
# means no registered prefix
archive = self.get_archive()
assert archive is not None
v_id = archive.find_value_id_for_alias(alias)
else:
v_id = archive.find_value_id_for_alias(alias=rest)
# TODO: cache this?
return v_id
def find_aliases_for_value_id(
self, value_id: uuid.UUID, search_dynamic_archives: bool = False
) -> Set[str]:
aliases = set([a.full_alias for a in self.aliases_by_id.get(value_id, [])])
if search_dynamic_archives:
for archive_alias, archive in self._alias_archives.items():
_aliases = archive.find_aliases_for_value_id(value_id=value_id)
# TODO: cache those results
if _aliases:
for a in _aliases:
aliases.add(f"{archive_alias}.{a}")
return aliases
def register_aliases(self, value_id: uuid.UUID, *aliases: str):
store_name = self.default_alias_store
store: AliasStore = self.get_archive(archive_id=store_name) # type: ignore
self.aliases # noqu
store.register_aliases(value_id, *aliases)
for alias in aliases:
alias_item = AliasItem(
full_alias=alias,
rel_alias=alias,
value_id=value_id,
alias_archive=store_name,
alias_archive_id=store.archive_id,
)
if alias in self.aliases.keys():
logger.info("alias.replace", alias=alias)
# raise NotImplementedError()
self.aliases[alias] = alias_item
self._cached_aliases_by_id.setdefault(value_id, set()).add(alias_item) # type: ignore
alias_archives: Mapping[str, kiara.registries.aliases.AliasArchive]
property
readonly
¶aliases: Dict[str, kiara.registries.aliases.AliasItem]
property
readonly
¶Retrieve a map of all available aliases, context wide, with the registered archive aliases as values.
aliases_by_id: Mapping[uuid.UUID, Set[kiara.registries.aliases.AliasItem]]
property
readonly
¶all_aliases: Iterable[str]
property
readonly
¶default_alias_store: str
property
readonly
¶find_aliases_for_value_id(self, value_id, search_dynamic_archives=False)
¶Source code in kiara/registries/aliases/__init__.py
def find_aliases_for_value_id(
self, value_id: uuid.UUID, search_dynamic_archives: bool = False
) -> Set[str]:
aliases = set([a.full_alias for a in self.aliases_by_id.get(value_id, [])])
if search_dynamic_archives:
for archive_alias, archive in self._alias_archives.items():
_aliases = archive.find_aliases_for_value_id(value_id=value_id)
# TODO: cache those results
if _aliases:
for a in _aliases:
aliases.add(f"{archive_alias}.{a}")
return aliases
find_value_id_for_alias(self, alias)
¶Source code in kiara/registries/aliases/__init__.py
def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]:
alias_item = self.aliases.get(alias, None)
if alias_item is not None:
return alias_item.value_id
if "." not in alias:
return None
archive_id, rest = alias.split(".", maxsplit=2)
archive = self.get_archive(archive_id=archive_id)
if archive is None:
# means no registered prefix
archive = self.get_archive()
assert archive is not None
v_id = archive.find_value_id_for_alias(alias)
else:
v_id = archive.find_value_id_for_alias(alias=rest)
# TODO: cache this?
return v_id
get_archive(self, archive_id=None)
¶Source code in kiara/registries/aliases/__init__.py
def get_archive(
self, archive_id: Union[str, None] = None
) -> Union[AliasArchive, None]:
if archive_id is None:
archive_id = self.default_alias_store
if archive_id is None:
raise Exception("Can't retrieve default alias archive, none set (yet).")
archive = self._alias_archives.get(archive_id, None)
return archive
register_aliases(self, value_id, *aliases)
¶Source code in kiara/registries/aliases/__init__.py
def register_aliases(self, value_id: uuid.UUID, *aliases: str):
store_name = self.default_alias_store
store: AliasStore = self.get_archive(archive_id=store_name) # type: ignore
self.aliases # noqu
store.register_aliases(value_id, *aliases)
for alias in aliases:
alias_item = AliasItem(
full_alias=alias,
rel_alias=alias,
value_id=value_id,
alias_archive=store_name,
alias_archive_id=store.archive_id,
)
if alias in self.aliases.keys():
logger.info("alias.replace", alias=alias)
# raise NotImplementedError()
self.aliases[alias] = alias_item
self._cached_aliases_by_id.setdefault(value_id, set()).add(alias_item) # type: ignore
register_archive(self, archive, alias=None, set_as_default_store=None)
¶Source code in kiara/registries/aliases/__init__.py
def register_archive(
self,
archive: AliasArchive,
alias: str = None,
set_as_default_store: Union[bool, None] = None,
):
alias_archive_id = archive.archive_id
archive.register_archive(kiara=self._kiara)
if alias is None:
alias = str(alias_archive_id)
if "." in alias:
raise Exception(
f"Can't register alias archive with as '{alias}': registered name is not allowed to contain a '.' character (yet)."
)
if alias in self._alias_archives.keys():
raise Exception(f"Can't add store, alias '{alias}' already registered.")
self._alias_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, AliasStore):
is_store = True
if set_as_default_store and self._default_alias_store is not None:
raise Exception(
f"Can't set alias store '{alias}' as default store: default store already set."
)
if self._default_alias_store is None:
is_default_store = True
self._default_alias_store = alias
event = AliasArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
alias_archive_id=archive.archive_id,
alias_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
AliasStore (AliasArchive)
¶Source code in kiara/registries/aliases/__init__.py
class AliasStore(AliasArchive):
@abc.abstractmethod
def register_aliases(self, value_id: uuid.UUID, *aliases: str):
pass
register_aliases(self, value_id, *aliases)
¶Source code in kiara/registries/aliases/__init__.py
@abc.abstractmethod
def register_aliases(self, value_id: uuid.UUID, *aliases: str):
pass
Modules¶
archives
¶
FileSystemAliasArchive (AliasArchive)
¶Source code in kiara/registries/aliases/archives.py
class FileSystemAliasArchive(AliasArchive):
_archive_type_name = "filesystem_alias_archive"
_config_cls = FileSystemArchiveConfig
def __init__(self, archive_id: uuid.UUID, config: ARCHIVE_CONFIG_CLS):
super().__init__(archive_id=archive_id, config=config)
self._base_path: Union[Path, None] = None
@property
def alias_store_path(self) -> Path:
if self._base_path is not None:
return self._base_path
self._base_path = Path(self.config.archive_path).absolute() # type: ignore
self._base_path.mkdir(parents=True, exist_ok=True)
return self._base_path
@property
def aliases_path(self) -> Path:
return self.alias_store_path / "aliases"
@property
def value_id_path(self) -> Path:
return self.alias_store_path / "value_ids"
def _delete_archive(self):
shutil.rmtree(self.alias_store_path)
def _translate_alias(self, alias: str) -> Path:
if "." in alias:
tokens = alias.split(".")
alias_path = (
self.aliases_path.joinpath(*tokens[0:-1]) / f"{tokens[-1]}.alias"
)
else:
alias_path = self.aliases_path / f"{alias}.alias"
return alias_path
def _translate_alias_path(self, alias_path: Path) -> str:
relative = (
alias_path.absolute()
.relative_to(self.aliases_path.absolute())
.as_posix()[:-6]
)
relative = os.path.normpath(relative)
if os.path.sep not in relative:
alias = relative
else:
alias = ".".join(relative.split(os.path.sep))
return alias
def _translate_value_id(self, value_id: uuid.UUID) -> Path:
tokens = str(value_id).split("-")
value_id_path = (
self.value_id_path.joinpath(*tokens[0:-1]) / f"{tokens[-1]}.value"
)
return value_id_path
def _translate_value_path(self, value_path: Path) -> uuid.UUID:
relative = (
value_path.absolute()
.relative_to(self.value_id_path.absolute())
.as_posix()[:-6]
)
relative = os.path.normpath(relative)
value_id_str = "-".join(relative.split(os.path.sep))
return uuid.UUID(value_id_str)
def retrieve_all_aliases(self) -> Mapping[str, uuid.UUID]:
all_aliases = self.aliases_path.rglob("*.alias")
result = {}
for alias_path in all_aliases:
alias = self._translate_alias_path(alias_path=alias_path)
value_id = self._find_value_id_for_alias_path(alias_path=alias_path)
assert value_id is not None
result[alias] = value_id
return result
def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]:
alias_path = self._translate_alias(alias)
if not alias_path.exists():
return None
return self._find_value_id_for_alias_path(alias_path=alias_path)
def _find_value_id_for_alias_path(self, alias_path: Path) -> Union[uuid.UUID, None]:
resolved = alias_path.resolve()
assert resolved.name.endswith(".value")
value_id = self._translate_value_path(value_path=resolved)
return value_id
def find_aliases_for_value_id(self, value_id: uuid.UUID) -> Union[Set[str], None]:
raise NotImplementedError()
alias_store_path: Path
property
readonly
¶aliases_path: Path
property
readonly
¶value_id_path: Path
property
readonly
¶
_config_cls (ArchiveConfig)
private
pydantic-model
¶Source code in kiara/registries/aliases/archives.py
class FileSystemArchiveConfig(ArchiveConfig):
archive_path: str = Field(
description="The path where the data for this archive is stored."
)
find_aliases_for_value_id(self, value_id)
¶Source code in kiara/registries/aliases/archives.py
def find_aliases_for_value_id(self, value_id: uuid.UUID) -> Union[Set[str], None]:
raise NotImplementedError()
find_value_id_for_alias(self, alias)
¶Source code in kiara/registries/aliases/archives.py
def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]:
alias_path = self._translate_alias(alias)
if not alias_path.exists():
return None
return self._find_value_id_for_alias_path(alias_path=alias_path)
retrieve_all_aliases(self)
¶Retrieve a list of all aliases registered in this archive.
The result of this method can be 'None', for cases where the aliases are determined dynamically. In kiara, the result of this method is mostly used to improve performance when looking up an alias.
Returns:
| Type | Description |
|---|---|
Mapping[str, uuid.UUID] |
a list of strings (the aliases), or 'None' if this archive does not support alias indexes. |
Source code in kiara/registries/aliases/archives.py
def retrieve_all_aliases(self) -> Mapping[str, uuid.UUID]:
all_aliases = self.aliases_path.rglob("*.alias")
result = {}
for alias_path in all_aliases:
alias = self._translate_alias_path(alias_path=alias_path)
value_id = self._find_value_id_for_alias_path(alias_path=alias_path)
assert value_id is not None
result[alias] = value_id
return result
FileSystemAliasStore (FileSystemAliasArchive, AliasStore)
¶Source code in kiara/registries/aliases/archives.py
class FileSystemAliasStore(FileSystemAliasArchive, AliasStore):
_archive_type_name = "filesystem_alias_store"
@classmethod
def is_writeable(cls) -> bool:
return True
def register_aliases(self, value_id: uuid.UUID, *aliases: str):
value_path = self._translate_value_id(value_id=value_id)
value_path.parent.mkdir(parents=True, exist_ok=True)
value_path.touch()
for alias in aliases:
alias_path = self._translate_alias(alias)
alias_path.parent.mkdir(parents=True, exist_ok=True)
if alias_path.exists():
resolved = alias_path.resolve()
if resolved == value_path:
continue
alias_path.unlink()
alias_path.symlink_to(value_path)
is_writeable()
classmethod
¶Source code in kiara/registries/aliases/archives.py
@classmethod
def is_writeable(cls) -> bool:
return True
register_aliases(self, value_id, *aliases)
¶Source code in kiara/registries/aliases/archives.py
def register_aliases(self, value_id: uuid.UUID, *aliases: str):
value_path = self._translate_value_id(value_id=value_id)
value_path.parent.mkdir(parents=True, exist_ok=True)
value_path.touch()
for alias in aliases:
alias_path = self._translate_alias(alias)
alias_path.parent.mkdir(parents=True, exist_ok=True)
if alias_path.exists():
resolved = alias_path.resolve()
if resolved == value_path:
continue
alias_path.unlink()
alias_path.symlink_to(value_path)
data
special
¶
NONE_PERSISTED_DATA
¶logger
¶Classes¶
AliasResolver (ABC)
¶Source code in kiara/registries/data/__init__.py
class AliasResolver(abc.ABC):
def __init__(self, kiara: "Kiara"):
self._kiara: "Kiara" = kiara
@abc.abstractmethod
def resolve_alias(self, alias: str) -> uuid.UUID:
pass
resolve_alias(self, alias)
¶Source code in kiara/registries/data/__init__.py
@abc.abstractmethod
def resolve_alias(self, alias: str) -> uuid.UUID:
pass
DataRegistry
¶Source code in kiara/registries/data/__init__.py
class DataRegistry(object):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._event_callback: Callable = self._kiara.event_registry.add_producer(self)
self._data_archives: Dict[str, DataArchive] = {}
self._default_data_store: Union[str, None] = None
self._registered_values: Dict[uuid.UUID, Value] = {}
self._value_archive_lookup_map: Dict[uuid.UUID, str] = {}
"""A cached dict that stores which archives which value ids belong to."""
self._values_by_hash: Dict[str, Set[uuid.UUID]] = {}
self._cached_data: Dict[uuid.UUID, Any] = {}
self._persisted_value_descs: Dict[uuid.UUID, Union[PersistedData, None]] = {}
self._alias_resolver: AliasResolver = DefaultAliasResolver(kiara=self._kiara)
# initialize special values
special_value_cls = PythonClass.from_class(NoneType)
data_type_info = DataTypeInfo.construct(
data_type_name="none",
characteristics=DEFAULT_SCALAR_DATATYPE_CHARACTERISTICS,
data_type_class=special_value_cls,
)
self._not_set_value: Value = Value(
value_id=NOT_SET_VALUE_ID,
kiara_id=self._kiara.id,
value_schema=ValueSchema(
type="none",
default=SpecialValue.NOT_SET,
is_constant=True,
doc="Special value, indicating a field is not set.", # type: ignore
),
environment_hashes={},
value_status=ValueStatus.NOT_SET,
value_size=0,
value_hash=INVALID_HASH_MARKER,
pedigree=ORPHAN,
pedigree_output_name="__void__",
data_type_info=data_type_info,
)
self._not_set_value._data_registry = self
self._cached_data[NOT_SET_VALUE_ID] = SpecialValue.NOT_SET
self._registered_values[NOT_SET_VALUE_ID] = self._not_set_value
self._persisted_value_descs[NOT_SET_VALUE_ID] = NONE_PERSISTED_DATA
self._none_value: Value = Value(
value_id=NONE_VALUE_ID,
kiara_id=self._kiara.id,
value_schema=ValueSchema(
type="none",
default=SpecialValue.NO_VALUE,
is_constant=True,
doc="Special value, indicating a field is set with a 'none' value.", # type: ignore
),
environment_hashes={},
value_status=ValueStatus.NONE,
value_size=0,
value_hash=str(NONE_CID),
pedigree=ORPHAN,
pedigree_output_name="__void__",
data_type_info=data_type_info,
)
self._none_value._data_registry = self
self._cached_data[NONE_VALUE_ID] = SpecialValue.NO_VALUE
self._registered_values[NONE_VALUE_ID] = self._none_value
self._persisted_value_descs[NONE_VALUE_ID] = NONE_PERSISTED_DATA
@property
def kiara_id(self) -> uuid.UUID:
return self._kiara.id
@property
def NOT_SET_VALUE(self) -> Value:
return self._not_set_value
@property
def NONE_VALUE(self) -> Value:
return self._none_value
def retrieve_all_available_value_ids(self) -> Set[uuid.UUID]:
result: Set[uuid.UUID] = set()
for store in self._data_archives.values():
ids = store.value_ids
if ids:
result.update(ids)
return result
def register_data_archive(
self,
archive: DataArchive,
alias: str = None,
set_as_default_store: Union[bool, None] = None,
):
data_store_id = archive.archive_id
archive.register_archive(kiara=self._kiara)
if alias is None:
alias = str(data_store_id)
if alias in self._data_archives.keys():
raise Exception(
f"Can't add data archive, alias '{alias}' already registered."
)
self._data_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, DataStore):
is_store = True
if set_as_default_store and self._default_data_store is not None:
raise Exception(
f"Can't set data store '{alias}' as default store: default store already set."
)
if self._default_data_store is None or set_as_default_store:
is_default_store = True
self._default_data_store = alias
event = DataArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
data_archive_id=archive.archive_id,
data_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
@property
def default_data_store(self) -> str:
if self._default_data_store is None:
raise Exception("No default data store set.")
return self._default_data_store
@property
def data_archives(self) -> Mapping[str, DataArchive]:
return self._data_archives
def get_archive(
self, archive_id: Union[None, uuid.UUID, str] = None
) -> DataArchive:
if archive_id is None:
archive_id = self.default_data_store
if archive_id is None:
raise Exception("Can't retrieve default data archive, none set (yet).")
if isinstance(archive_id, uuid.UUID):
for archive in self._data_archives.values():
if archive.archive_id == archive_id:
return archive
raise Exception(
f"Can't retrieve archive with id '{archive_id}': no archive with that id registered."
)
if archive_id in self._data_archives.keys():
return self._data_archives[archive_id]
else:
try:
_archive_id = uuid.UUID(archive_id)
for archive in self._data_archives.values():
if archive.archive_id == _archive_id:
return archive
raise Exception(
f"Can't retrieve archive with id '{archive_id}': no archive with that id registered."
)
except Exception:
pass
raise Exception(
f"Can't retrieve archive with id '{archive_id}': no archive with that id registered."
)
def find_store_id_for_value(self, value_id: uuid.UUID) -> Union[str, None]:
if value_id in self._value_archive_lookup_map.keys():
return self._value_archive_lookup_map[value_id]
matches = []
for store_id, store in self.data_archives.items():
match = store.has_value(value_id=value_id)
if match:
matches.append(store_id)
if len(matches) == 0:
return None
elif len(matches) > 1:
raise Exception(
f"Found value with id '{value_id}' in multiple archives, this is not supported (yet): {matches}"
)
self._value_archive_lookup_map[value_id] = matches[0]
return matches[0]
def get_value(self, value: Union[uuid.UUID, ValueLink, str]) -> Value:
_value_id = None
if not isinstance(value, uuid.UUID):
# fallbacks for common mistakes, this should error out if not a Value or string.
if hasattr(value, "value_id"):
_value_id: Union[uuid.UUID, str, None] = value.value_id # type: ignore
if isinstance(_value_id, str):
_value_id = uuid.UUID(_value_id)
else:
try:
_value_id = uuid.UUID(
value # type: ignore
) # this should fail if not string or wrong string format
except ValueError:
_value_id = None
if _value_id is None:
if not isinstance(value, str):
raise Exception(
f"Can't retrieve value for '{value}': invalid type '{type(value)}'."
)
_value_id = self._alias_resolver.resolve_alias(value)
else:
_value_id = value
assert _value_id is not None
if _value_id in self._registered_values.keys():
_value = self._registered_values[_value_id]
return _value
matches = []
for store_id, store in self.data_archives.items():
match = store.has_value(value_id=_value_id)
if match:
matches.append(store_id)
if len(matches) == 0:
raise NoSuchValueIdException(
value_id=_value_id, msg=f"No value registered with id: {value}"
)
elif len(matches) > 1:
raise NoSuchValueIdException(
value_id=_value_id,
msg=f"Found value with id '{value}' in multiple archives, this is not supported (yet): {matches}",
)
self._value_archive_lookup_map[_value_id] = matches[0]
stored_value = self.get_archive(matches[0]).retrieve_value(value_id=_value_id)
stored_value._set_registry(self)
stored_value._is_stored = True
self._registered_values[_value_id] = stored_value
return self._registered_values[_value_id]
def store_value(
self,
value: Union[ValueLink, uuid.UUID, str],
store_id: Union[str, None] = None,
) -> Union[PersistedData, None]:
if store_id is None:
store_id = self.default_data_store
_value = self.get_value(value)
store: DataStore = self.get_archive(archive_id=store_id) # type: ignore
if not isinstance(store, DataStore):
raise Exception(f"Can't store value into store '{store_id}': not writable.")
# make sure all property values are available
if _value.pedigree != ORPHAN:
for value_id in _value.pedigree.inputs.values():
self.store_value(value=value_id, store_id=store_id)
if not store.has_value(_value.value_id):
event = ValuePreStoreEvent.construct(kiara_id=self._kiara.id, value=_value)
self._event_callback(event)
persisted_value = store.store_value(_value)
_value._is_stored = True
self._value_archive_lookup_map[_value.value_id] = store_id
self._persisted_value_descs[_value.value_id] = persisted_value
property_values = _value.property_values
for property, property_value in property_values.items():
self.store_value(value=property_value, store_id=store_id)
else:
persisted_value = None
store_event = ValueStoredEvent.construct(kiara_id=self._kiara.id, value=_value)
self._event_callback(store_event)
return persisted_value
def lookup_aliases(self, value: Union[Value, uuid.UUID]) -> Set[str]:
if isinstance(value, Value):
value = value.value_id
return self._kiara.alias_registry.find_aliases_for_value_id(value_id=value)
def create_value_info(self, value: Union[Value, uuid.UUID]) -> ValueInfo:
if isinstance(value, uuid.UUID):
value = self.get_value(value=value)
value_info = ValueInfo.create_from_instance(kiara=self._kiara, instance=value)
return value_info
def find_values(self, matcher: ValueMatcher) -> Dict[uuid.UUID, Value]:
matches: Dict[uuid.UUID, Value] = {}
for store_id, store in self.data_archives.items():
try:
_matches = store.find_values(matcher=matcher)
for value in _matches:
if value.value_id in matches.keys():
raise Exception(
f"Found value '{value.value_id}' multiple times, this is not supported yet."
)
self._value_archive_lookup_map[value.value_id] = store_id
value._set_registry(self)
value._is_stored = True
self._registered_values[value.value_id] = value
matches[value.value_id] = value
self._values_by_hash.setdefault(value.value_hash, set()).add(
value.value_id
)
except NotImplementedError:
all_value_ids = store.value_ids
if all_value_ids is None:
continue
for value_id in all_value_ids:
value = store.retrieve_value(value_id=value_id)
value._set_registry(self)
value._is_stored = True
self._registered_values[value.value_id] = value
match = matcher.is_match(value, kiara=self._kiara)
if match:
if value.value_id in matches.keys():
raise Exception(
f"Found value '{value.value_id}' multiple times, this is not supported yet."
)
matches[value.value_id] = value
return matches
def find_values_with_aliases(self, matcher: ValueMatcher) -> Dict[str, Value]:
matcher = matcher.copy(update={"has_aliases": True})
all_values = self.find_values(matcher)
result = {}
for value in all_values.values():
aliases = self._kiara.alias_registry.find_aliases_for_value_id(
value_id=value.value_id
)
for a in aliases:
assert a not in result # this is a bug
result[a] = value
return result
def find_values_for_hash(
self, value_hash: str, data_type_name: Union[str, None] = None
) -> Set[Value]:
if data_type_name:
raise NotImplementedError()
stored = self._values_by_hash.get(value_hash, None)
if stored is None:
matches: Dict[uuid.UUID, List[str]] = {}
for store_id, store in self.data_archives.items():
value_ids = store.find_values_with_hash(
value_hash=value_hash, data_type_name=data_type_name
)
for v_id in value_ids:
matches.setdefault(v_id, []).append(store_id)
stored = set()
for v_id, store_ids in matches.items():
if len(store_ids) > 1:
raise Exception(
f"Found multiple stores for value id '{v_id}', this is not supported (yet)."
)
self._value_archive_lookup_map[v_id] = store_ids[0]
stored.add(v_id)
if stored:
self._values_by_hash[value_hash] = stored
return set((self.get_value(value=v_id) for v_id in stored))
def find_destinies_for_value(
self, value_id: uuid.UUID, alias_filter: str = None
) -> Mapping[str, uuid.UUID]:
if alias_filter:
raise NotImplementedError()
all_destinies: Dict[str, uuid.UUID] = {}
for archive_id, archive in self._data_archives.items():
destinies: Union[
Mapping[str, uuid.UUID], None
] = archive.find_destinies_for_value(
value_id=value_id, alias_filter=alias_filter
)
if not destinies:
continue
for k, v in destinies.items():
if k in all_destinies.keys():
raise Exception(f"Duplicate destiny '{k}' for value '{value_id}'.")
all_destinies[k] = v
return all_destinies
def register_data(
self,
data: Any,
schema: Union[ValueSchema, str] = None,
pedigree: Union[ValuePedigree, None] = None,
pedigree_output_name: str = None,
reuse_existing: bool = True,
) -> Value:
value, newly_created = self._create_value(
data=data,
schema=schema,
pedigree=pedigree,
pedigree_output_name=pedigree_output_name,
reuse_existing=reuse_existing,
)
if newly_created:
self._values_by_hash.setdefault(value.value_hash, set()).add(value.value_id)
self._registered_values[value.value_id] = value
self._cached_data[value.value_id] = data
event = ValueRegisteredEvent(kiara_id=self._kiara.id, value=value)
self._event_callback(event)
return value
def _find_existing_value(
self, data: Any, schema: Union[ValueSchema, None]
) -> Tuple[
Union[Value, None],
DataType,
Union[Any, None],
Union[str, SerializedData],
ValueStatus,
str,
int,
]:
if schema is None:
raise NotImplementedError()
if isinstance(data, Value):
if data.value_id in self._registered_values.keys():
if data.is_set and data.is_serializable:
serialized: Union[str, SerializedData] = data.serialized_data
else:
serialized = NO_SERIALIZATION_MARKER
return (
data,
data.data_type,
None,
serialized,
data.value_status,
data.value_hash,
data.value_size,
)
raise NotImplementedError("Importing values not supported (yet).")
# self._registered_values[data.value_id] = data
# return data
try:
value = self.get_value(value=data)
if value.is_serializable:
serialized = value.serialized_data
else:
serialized = NO_SERIALIZATION_MARKER
return (
value,
value.data_type,
None,
serialized,
value.value_status,
value.value_hash,
value.value_size,
)
except NoSuchValueException as nsve:
raise nsve
except Exception:
# TODO: differentiate between 'value not found' and other type of errors
pass
# no obvious matches, so we try to find data that has the same hash
data_type = self._kiara.type_registry.retrieve_data_type(
data_type_name=schema.type, data_type_config=schema.type_config
)
data, serialized, status, value_hash, value_size = data_type._pre_examine_data(
data=data, schema=schema
)
existing_value: Union[Value, None] = None
if value_hash != INVALID_HASH_MARKER:
existing = self.find_values_for_hash(value_hash=value_hash)
if existing:
if len(existing) == 1:
existing_value = next(iter(existing))
else:
skalars = []
for v in existing:
if v.data_type.characteristics.is_scalar:
skalars.append(v)
if len(skalars) == 1:
existing_value = skalars[0]
elif skalars:
orphans = []
for v in skalars:
if v.pedigree == ORPHAN:
orphans.append(v)
if len(orphans) == 1:
existing_value = orphans[0]
if existing_value is not None:
self._persisted_value_descs[existing_value.value_id] = None
return (
existing_value,
data_type,
data,
serialized,
status,
value_hash,
value_size,
)
return (None, data_type, data, serialized, status, value_hash, value_size)
def _create_value(
self,
data: Any,
schema: Union[None, str, ValueSchema] = None,
pedigree: Union[ValuePedigree, None] = None,
pedigree_output_name: str = None,
reuse_existing: bool = True,
) -> Tuple[Value, bool]:
"""Create a new value, or return an existing one that matches the incoming data or reference.
Arguments:
data: the (raw) data, or a reference to an existing value
Returns:
a tuple containing of the value object, and a boolean indicating whether the value was newly created (True), or already existing (False)
"""
if schema is None:
raise NotImplementedError()
elif isinstance(schema, str):
schema = ValueSchema(type=schema)
if schema.type not in self._kiara.data_type_names:
raise Exception(
f"Can't register data of type '{schema.type}': type not registered. Available types: {', '.join(self._kiara.data_type_names)}"
)
if data is SpecialValue.NOT_SET and schema.default is not SpecialValue.NOT_SET:
if callable(schema.default):
raise NotImplementedError()
data = schema.default()
else:
data = copy.deepcopy(schema.default)
if data is None:
data = SpecialValue.NO_VALUE
if reuse_existing and data not in [
SpecialValue.NO_VALUE,
SpecialValue.NOT_SET,
]:
(
_existing,
data_type,
data,
serialized,
status,
value_hash,
value_size,
) = self._find_existing_value(data=data, schema=schema)
if _existing is not None:
# TODO: check pedigree
return (_existing, False)
else:
data_type = self._kiara.type_registry.retrieve_data_type(
data_type_name=schema.type, data_type_config=schema.type_config
)
(
data,
serialized,
status,
value_hash,
value_size,
) = data_type._pre_examine_data(data=data, schema=schema)
if pedigree is None:
pedigree = ORPHAN
if pedigree_output_name is None:
if pedigree == ORPHAN:
pedigree_output_name = ORPHAN_PEDIGREE_OUTPUT_NAME
else:
raise NotImplementedError()
v_id = ID_REGISTRY.generate(
type="value", kiara_id=self._kiara.id, pre_registered=False
)
value, data = data_type.assemble_value(
value_id=v_id,
data=data,
schema=schema,
environment_hashes=self._kiara.environment_registry.environment_hashes,
serialized=serialized,
status=status,
value_hash=value_hash,
value_size=value_size,
pedigree=pedigree,
kiara_id=self._kiara.id,
pedigree_output_name=pedigree_output_name,
)
ID_REGISTRY.update_metadata(v_id, obj=value)
value._data_registry = self
event = ValueCreatedEvent(kiara_id=self._kiara.id, value=value)
self._event_callback(event)
return (value, True)
def retrieve_persisted_value_details(self, value_id: uuid.UUID) -> PersistedData:
if (
value_id in self._persisted_value_descs.keys()
and self._persisted_value_descs[value_id] is not None
):
persisted_details = self._persisted_value_descs[value_id]
assert persisted_details is not None
else:
# now, the value_store map should contain this value_id
store_id = self.find_store_id_for_value(value_id=value_id)
if store_id is None:
raise Exception(
f"Can't find store for persisted data of value: {value_id}"
)
store = self.get_archive(store_id)
assert value_id in self._registered_values.keys()
# self.get_value(value_id=value_id)
persisted_details = store.retrieve_serialized_value(value=value_id)
for c in persisted_details.chunk_id_map.values():
c._data_registry = self._kiara.data_registry
self._persisted_value_descs[value_id] = persisted_details
return persisted_details
# def _retrieve_bytes(
# self, chunk_id: str, as_link: bool = True
# ) -> Union[str, bytes]:
#
# # TODO: support multiple stores
# return self.get_archive().retrieve_chunk(chunk_id=chunk_id, as_link=as_link)
def retrieve_serialized_value(
self, value_id: uuid.UUID
) -> Union[SerializedData, None]:
"""Create a LoadConfig object from the details of the persisted version of this value."""
pv = self.retrieve_persisted_value_details(value_id=value_id)
if pv is None:
return None
return pv
def retrieve_chunk(
self,
chunk_id: str,
archive_id: Union[uuid.UUID, None] = None,
as_file: Union[None, bool, str] = None,
symlink_ok: bool = True,
) -> Union[str, bytes]:
if archive_id is None:
raise NotImplementedError()
archive = self.get_archive(archive_id)
chunk = archive.retrieve_chunk(chunk_id, as_file=as_file, symlink_ok=symlink_ok)
return chunk
def retrieve_value_data(
self, value: Union[uuid.UUID, Value], target_profile: Union[str, None] = None
) -> Any:
if isinstance(value, uuid.UUID):
value = self.get_value(value=value)
if value.value_id in self._cached_data.keys():
return self._cached_data[value.value_id]
if value._serialized_data is None:
serialized_data: Union[
str, SerializedData
] = self.retrieve_persisted_value_details(value_id=value.value_id)
value._serialized_data = serialized_data
else:
serialized_data = value._serialized_data
if isinstance(serialized_data, str):
raise Exception(
f"Can't retrieve serialized version of value '{value.value_id}', this is most likely a bug."
)
manifest = serialized_data.metadata.deserialize.get("python_object", None)
if manifest is None:
raise Exception(
f"No deserialize operation found for data type: {value.data_type_name}"
)
module = self._kiara.create_module(manifest=manifest)
op = Operation.create_from_module(module=module)
input_field_match: Union[str, None] = None
if len(op.inputs_schema) == 1:
input_field_match = next(iter(op.inputs_schema.keys()))
else:
for input_field, schema in op.inputs_schema.items():
for dt in self._kiara.type_registry.get_type_lineage(
value.data_type_name
):
if schema.type == dt:
if input_field_match is not None:
raise Exception(
f"Can't determine input field for deserialization operation '{module.module_type_name}': multiple input fields with type '{input_field_match}'."
)
else:
input_field_match = input_field
break
if input_field_match:
break
if input_field_match is None:
raise Exception(
f"Can't determine input field for deserialization operation '{module.module_type_name}'."
)
result_field_match: Union[str, None] = None
for result_field, schema in op.outputs_schema.items():
if schema.type == "python_object":
if result_field_match is not None:
raise Exception(
f"Can't determine result field for deserialization operation '{module.module_type_name}': multiple result fields with type 'python_object'."
)
else:
result_field_match = result_field
if result_field_match is None:
raise Exception(
f"Can't determine result field for deserialization operation '{module.module_type_name}'."
)
inputs = {input_field_match: value}
result = op.run(kiara=self._kiara, inputs=inputs)
python_object = result.get_value_data(result_field_match)
# TODO: can we do without this?
parsed = value.data_type.parse_python_obj(python_object)
value.data_type._validate(parsed)
self._cached_data[value.value_id] = parsed
return parsed
def load_values(self, values: Mapping[str, Union[uuid.UUID, None]]) -> ValueMap:
value_items = {}
schemas = {}
for field_name, value_id in values.items():
if value_id is None:
value_id = NONE_VALUE_ID
value = self.get_value(value=value_id)
value_items[field_name] = value
schemas[field_name] = value.value_schema
return ValueMapReadOnly(value_items=value_items, values_schema=schemas)
def load_data(
self, values: Mapping[str, Union[uuid.UUID, None]]
) -> Mapping[str, Any]:
result_values = self.load_values(values=values)
return {k: v.data for k, v in result_values.items()}
def create_valuemap(
self, data: Mapping[str, Any], schema: Mapping[str, ValueSchema]
) -> ValueMap:
"""Extract a set of [Value][kiara.data.values.Value] from Python data and ValueSchemas."""
input_details = {}
for input_name, value_schema in schema.items():
input_details[input_name] = {"schema": value_schema}
leftover = set(data.keys())
leftover.difference_update(input_details.keys())
if leftover:
if not STRICT_CHECKS:
log_message("unused.inputs", input_names=leftover)
else:
raise Exception(
f"Can't create values instance, inputs contain unused/invalid fields: {', '.join(leftover)}"
)
values = {}
failed = {}
_resolved: Dict[str, Value] = {}
_unresolved = {}
for input_name, details in input_details.items():
_d = data.get(input_name, SpecialValue.NOT_SET)
if isinstance(_d, str):
try:
_d = uuid.UUID(_d)
except Exception:
pass
if isinstance(_d, Value):
_resolved[input_name] = _d
elif isinstance(_d, uuid.UUID):
_resolved[input_name] = self.get_value(_d)
else:
_unresolved[input_name] = _d
for input_name, _value in _resolved.items():
# TODO: validate values against schema
values[input_name] = _value
for input_name, _data in _unresolved.items():
value_schema = input_details[input_name]["schema"]
if input_name not in data.keys():
value_data = SpecialValue.NOT_SET
elif data[input_name] in [
None,
SpecialValue.NO_VALUE,
SpecialValue.NOT_SET,
]:
value_data = SpecialValue.NO_VALUE
else:
value_data = data[input_name]
try:
value = self.register_data(
data=value_data, schema=value_schema, reuse_existing=True
)
values[input_name] = value
except Exception as e:
log_exception(e)
msg: Any = str(e)
if not msg:
msg = e
log_message("invalid.valueset", error_reason=msg, input_name=input_name)
failed[input_name] = e
if failed:
msg = []
for k, v in failed.items():
_v = str(v)
if not str(v):
_v = type(v).__name__
msg.append(f"{k}: {_v}")
raise InvalidValuesException(
msg=f"Can't create values instance: {', '.join(msg)}",
invalid_values={k: str(v) for k, v in failed.items()},
)
return ValueMapReadOnly(value_items=values, values_schema=schema) # type: ignore
def create_renderable(self, **config: Any) -> RenderableType:
"""Create a renderable for this module configuration."""
from kiara.utils.output import create_renderable_from_values
all_values = {str(i): v for i, v in self._registered_values.items()}
table = create_renderable_from_values(values=all_values, config=config)
return table
def pretty_print_data(
self,
value_id: uuid.UUID,
target_type="terminal_renderable",
**render_config: Any,
) -> Any:
assert isinstance(value_id, uuid.UUID)
return pretty_print_data(
kiara=self._kiara,
value_id=value_id,
target_type=target_type,
**render_config,
)
NONE_VALUE: Value
property
readonly
¶NOT_SET_VALUE: Value
property
readonly
¶data_archives: Mapping[str, kiara.registries.data.data_store.DataArchive]
property
readonly
¶default_data_store: str
property
readonly
¶kiara_id: UUID
property
readonly
¶create_renderable(self, **config)
¶Create a renderable for this module configuration.
Source code in kiara/registries/data/__init__.py
def create_renderable(self, **config: Any) -> RenderableType:
"""Create a renderable for this module configuration."""
from kiara.utils.output import create_renderable_from_values
all_values = {str(i): v for i, v in self._registered_values.items()}
table = create_renderable_from_values(values=all_values, config=config)
return table
create_value_info(self, value)
¶Source code in kiara/registries/data/__init__.py
def create_value_info(self, value: Union[Value, uuid.UUID]) -> ValueInfo:
if isinstance(value, uuid.UUID):
value = self.get_value(value=value)
value_info = ValueInfo.create_from_instance(kiara=self._kiara, instance=value)
return value_info
create_valuemap(self, data, schema)
¶Extract a set of [Value][kiara.data.values.Value] from Python data and ValueSchemas.
Source code in kiara/registries/data/__init__.py
def create_valuemap(
self, data: Mapping[str, Any], schema: Mapping[str, ValueSchema]
) -> ValueMap:
"""Extract a set of [Value][kiara.data.values.Value] from Python data and ValueSchemas."""
input_details = {}
for input_name, value_schema in schema.items():
input_details[input_name] = {"schema": value_schema}
leftover = set(data.keys())
leftover.difference_update(input_details.keys())
if leftover:
if not STRICT_CHECKS:
log_message("unused.inputs", input_names=leftover)
else:
raise Exception(
f"Can't create values instance, inputs contain unused/invalid fields: {', '.join(leftover)}"
)
values = {}
failed = {}
_resolved: Dict[str, Value] = {}
_unresolved = {}
for input_name, details in input_details.items():
_d = data.get(input_name, SpecialValue.NOT_SET)
if isinstance(_d, str):
try:
_d = uuid.UUID(_d)
except Exception:
pass
if isinstance(_d, Value):
_resolved[input_name] = _d
elif isinstance(_d, uuid.UUID):
_resolved[input_name] = self.get_value(_d)
else:
_unresolved[input_name] = _d
for input_name, _value in _resolved.items():
# TODO: validate values against schema
values[input_name] = _value
for input_name, _data in _unresolved.items():
value_schema = input_details[input_name]["schema"]
if input_name not in data.keys():
value_data = SpecialValue.NOT_SET
elif data[input_name] in [
None,
SpecialValue.NO_VALUE,
SpecialValue.NOT_SET,
]:
value_data = SpecialValue.NO_VALUE
else:
value_data = data[input_name]
try:
value = self.register_data(
data=value_data, schema=value_schema, reuse_existing=True
)
values[input_name] = value
except Exception as e:
log_exception(e)
msg: Any = str(e)
if not msg:
msg = e
log_message("invalid.valueset", error_reason=msg, input_name=input_name)
failed[input_name] = e
if failed:
msg = []
for k, v in failed.items():
_v = str(v)
if not str(v):
_v = type(v).__name__
msg.append(f"{k}: {_v}")
raise InvalidValuesException(
msg=f"Can't create values instance: {', '.join(msg)}",
invalid_values={k: str(v) for k, v in failed.items()},
)
return ValueMapReadOnly(value_items=values, values_schema=schema) # type: ignore
find_destinies_for_value(self, value_id, alias_filter=None)
¶Source code in kiara/registries/data/__init__.py
def find_destinies_for_value(
self, value_id: uuid.UUID, alias_filter: str = None
) -> Mapping[str, uuid.UUID]:
if alias_filter:
raise NotImplementedError()
all_destinies: Dict[str, uuid.UUID] = {}
for archive_id, archive in self._data_archives.items():
destinies: Union[
Mapping[str, uuid.UUID], None
] = archive.find_destinies_for_value(
value_id=value_id, alias_filter=alias_filter
)
if not destinies:
continue
for k, v in destinies.items():
if k in all_destinies.keys():
raise Exception(f"Duplicate destiny '{k}' for value '{value_id}'.")
all_destinies[k] = v
return all_destinies
find_store_id_for_value(self, value_id)
¶Source code in kiara/registries/data/__init__.py
def find_store_id_for_value(self, value_id: uuid.UUID) -> Union[str, None]:
if value_id in self._value_archive_lookup_map.keys():
return self._value_archive_lookup_map[value_id]
matches = []
for store_id, store in self.data_archives.items():
match = store.has_value(value_id=value_id)
if match:
matches.append(store_id)
if len(matches) == 0:
return None
elif len(matches) > 1:
raise Exception(
f"Found value with id '{value_id}' in multiple archives, this is not supported (yet): {matches}"
)
self._value_archive_lookup_map[value_id] = matches[0]
return matches[0]
find_values(self, matcher)
¶Source code in kiara/registries/data/__init__.py
def find_values(self, matcher: ValueMatcher) -> Dict[uuid.UUID, Value]:
matches: Dict[uuid.UUID, Value] = {}
for store_id, store in self.data_archives.items():
try:
_matches = store.find_values(matcher=matcher)
for value in _matches:
if value.value_id in matches.keys():
raise Exception(
f"Found value '{value.value_id}' multiple times, this is not supported yet."
)
self._value_archive_lookup_map[value.value_id] = store_id
value._set_registry(self)
value._is_stored = True
self._registered_values[value.value_id] = value
matches[value.value_id] = value
self._values_by_hash.setdefault(value.value_hash, set()).add(
value.value_id
)
except NotImplementedError:
all_value_ids = store.value_ids
if all_value_ids is None:
continue
for value_id in all_value_ids:
value = store.retrieve_value(value_id=value_id)
value._set_registry(self)
value._is_stored = True
self._registered_values[value.value_id] = value
match = matcher.is_match(value, kiara=self._kiara)
if match:
if value.value_id in matches.keys():
raise Exception(
f"Found value '{value.value_id}' multiple times, this is not supported yet."
)
matches[value.value_id] = value
return matches
find_values_for_hash(self, value_hash, data_type_name=None)
¶Source code in kiara/registries/data/__init__.py
def find_values_for_hash(
self, value_hash: str, data_type_name: Union[str, None] = None
) -> Set[Value]:
if data_type_name:
raise NotImplementedError()
stored = self._values_by_hash.get(value_hash, None)
if stored is None:
matches: Dict[uuid.UUID, List[str]] = {}
for store_id, store in self.data_archives.items():
value_ids = store.find_values_with_hash(
value_hash=value_hash, data_type_name=data_type_name
)
for v_id in value_ids:
matches.setdefault(v_id, []).append(store_id)
stored = set()
for v_id, store_ids in matches.items():
if len(store_ids) > 1:
raise Exception(
f"Found multiple stores for value id '{v_id}', this is not supported (yet)."
)
self._value_archive_lookup_map[v_id] = store_ids[0]
stored.add(v_id)
if stored:
self._values_by_hash[value_hash] = stored
return set((self.get_value(value=v_id) for v_id in stored))
find_values_with_aliases(self, matcher)
¶Source code in kiara/registries/data/__init__.py
def find_values_with_aliases(self, matcher: ValueMatcher) -> Dict[str, Value]:
matcher = matcher.copy(update={"has_aliases": True})
all_values = self.find_values(matcher)
result = {}
for value in all_values.values():
aliases = self._kiara.alias_registry.find_aliases_for_value_id(
value_id=value.value_id
)
for a in aliases:
assert a not in result # this is a bug
result[a] = value
return result
get_archive(self, archive_id=None)
¶Source code in kiara/registries/data/__init__.py
def get_archive(
self, archive_id: Union[None, uuid.UUID, str] = None
) -> DataArchive:
if archive_id is None:
archive_id = self.default_data_store
if archive_id is None:
raise Exception("Can't retrieve default data archive, none set (yet).")
if isinstance(archive_id, uuid.UUID):
for archive in self._data_archives.values():
if archive.archive_id == archive_id:
return archive
raise Exception(
f"Can't retrieve archive with id '{archive_id}': no archive with that id registered."
)
if archive_id in self._data_archives.keys():
return self._data_archives[archive_id]
else:
try:
_archive_id = uuid.UUID(archive_id)
for archive in self._data_archives.values():
if archive.archive_id == _archive_id:
return archive
raise Exception(
f"Can't retrieve archive with id '{archive_id}': no archive with that id registered."
)
except Exception:
pass
raise Exception(
f"Can't retrieve archive with id '{archive_id}': no archive with that id registered."
)
get_value(self, value)
¶Source code in kiara/registries/data/__init__.py
def get_value(self, value: Union[uuid.UUID, ValueLink, str]) -> Value:
_value_id = None
if not isinstance(value, uuid.UUID):
# fallbacks for common mistakes, this should error out if not a Value or string.
if hasattr(value, "value_id"):
_value_id: Union[uuid.UUID, str, None] = value.value_id # type: ignore
if isinstance(_value_id, str):
_value_id = uuid.UUID(_value_id)
else:
try:
_value_id = uuid.UUID(
value # type: ignore
) # this should fail if not string or wrong string format
except ValueError:
_value_id = None
if _value_id is None:
if not isinstance(value, str):
raise Exception(
f"Can't retrieve value for '{value}': invalid type '{type(value)}'."
)
_value_id = self._alias_resolver.resolve_alias(value)
else:
_value_id = value
assert _value_id is not None
if _value_id in self._registered_values.keys():
_value = self._registered_values[_value_id]
return _value
matches = []
for store_id, store in self.data_archives.items():
match = store.has_value(value_id=_value_id)
if match:
matches.append(store_id)
if len(matches) == 0:
raise NoSuchValueIdException(
value_id=_value_id, msg=f"No value registered with id: {value}"
)
elif len(matches) > 1:
raise NoSuchValueIdException(
value_id=_value_id,
msg=f"Found value with id '{value}' in multiple archives, this is not supported (yet): {matches}",
)
self._value_archive_lookup_map[_value_id] = matches[0]
stored_value = self.get_archive(matches[0]).retrieve_value(value_id=_value_id)
stored_value._set_registry(self)
stored_value._is_stored = True
self._registered_values[_value_id] = stored_value
return self._registered_values[_value_id]
load_data(self, values)
¶Source code in kiara/registries/data/__init__.py
def load_data(
self, values: Mapping[str, Union[uuid.UUID, None]]
) -> Mapping[str, Any]:
result_values = self.load_values(values=values)
return {k: v.data for k, v in result_values.items()}
load_values(self, values)
¶Source code in kiara/registries/data/__init__.py
def load_values(self, values: Mapping[str, Union[uuid.UUID, None]]) -> ValueMap:
value_items = {}
schemas = {}
for field_name, value_id in values.items():
if value_id is None:
value_id = NONE_VALUE_ID
value = self.get_value(value=value_id)
value_items[field_name] = value
schemas[field_name] = value.value_schema
return ValueMapReadOnly(value_items=value_items, values_schema=schemas)
lookup_aliases(self, value)
¶Source code in kiara/registries/data/__init__.py
def lookup_aliases(self, value: Union[Value, uuid.UUID]) -> Set[str]:
if isinstance(value, Value):
value = value.value_id
return self._kiara.alias_registry.find_aliases_for_value_id(value_id=value)
pretty_print_data(self, value_id, target_type='terminal_renderable', **render_config)
¶Source code in kiara/registries/data/__init__.py
def pretty_print_data(
self,
value_id: uuid.UUID,
target_type="terminal_renderable",
**render_config: Any,
) -> Any:
assert isinstance(value_id, uuid.UUID)
return pretty_print_data(
kiara=self._kiara,
value_id=value_id,
target_type=target_type,
**render_config,
)
register_data(self, data, schema=None, pedigree=None, pedigree_output_name=None, reuse_existing=True)
¶Source code in kiara/registries/data/__init__.py
def register_data(
self,
data: Any,
schema: Union[ValueSchema, str] = None,
pedigree: Union[ValuePedigree, None] = None,
pedigree_output_name: str = None,
reuse_existing: bool = True,
) -> Value:
value, newly_created = self._create_value(
data=data,
schema=schema,
pedigree=pedigree,
pedigree_output_name=pedigree_output_name,
reuse_existing=reuse_existing,
)
if newly_created:
self._values_by_hash.setdefault(value.value_hash, set()).add(value.value_id)
self._registered_values[value.value_id] = value
self._cached_data[value.value_id] = data
event = ValueRegisteredEvent(kiara_id=self._kiara.id, value=value)
self._event_callback(event)
return value
register_data_archive(self, archive, alias=None, set_as_default_store=None)
¶Source code in kiara/registries/data/__init__.py
def register_data_archive(
self,
archive: DataArchive,
alias: str = None,
set_as_default_store: Union[bool, None] = None,
):
data_store_id = archive.archive_id
archive.register_archive(kiara=self._kiara)
if alias is None:
alias = str(data_store_id)
if alias in self._data_archives.keys():
raise Exception(
f"Can't add data archive, alias '{alias}' already registered."
)
self._data_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, DataStore):
is_store = True
if set_as_default_store and self._default_data_store is not None:
raise Exception(
f"Can't set data store '{alias}' as default store: default store already set."
)
if self._default_data_store is None or set_as_default_store:
is_default_store = True
self._default_data_store = alias
event = DataArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
data_archive_id=archive.archive_id,
data_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
retrieve_all_available_value_ids(self)
¶Source code in kiara/registries/data/__init__.py
def retrieve_all_available_value_ids(self) -> Set[uuid.UUID]:
result: Set[uuid.UUID] = set()
for store in self._data_archives.values():
ids = store.value_ids
if ids:
result.update(ids)
return result
retrieve_chunk(self, chunk_id, archive_id=None, as_file=None, symlink_ok=True)
¶Source code in kiara/registries/data/__init__.py
def retrieve_chunk(
self,
chunk_id: str,
archive_id: Union[uuid.UUID, None] = None,
as_file: Union[None, bool, str] = None,
symlink_ok: bool = True,
) -> Union[str, bytes]:
if archive_id is None:
raise NotImplementedError()
archive = self.get_archive(archive_id)
chunk = archive.retrieve_chunk(chunk_id, as_file=as_file, symlink_ok=symlink_ok)
return chunk
retrieve_persisted_value_details(self, value_id)
¶Source code in kiara/registries/data/__init__.py
def retrieve_persisted_value_details(self, value_id: uuid.UUID) -> PersistedData:
if (
value_id in self._persisted_value_descs.keys()
and self._persisted_value_descs[value_id] is not None
):
persisted_details = self._persisted_value_descs[value_id]
assert persisted_details is not None
else:
# now, the value_store map should contain this value_id
store_id = self.find_store_id_for_value(value_id=value_id)
if store_id is None:
raise Exception(
f"Can't find store for persisted data of value: {value_id}"
)
store = self.get_archive(store_id)
assert value_id in self._registered_values.keys()
# self.get_value(value_id=value_id)
persisted_details = store.retrieve_serialized_value(value=value_id)
for c in persisted_details.chunk_id_map.values():
c._data_registry = self._kiara.data_registry
self._persisted_value_descs[value_id] = persisted_details
return persisted_details
retrieve_serialized_value(self, value_id)
¶Create a LoadConfig object from the details of the persisted version of this value.
Source code in kiara/registries/data/__init__.py
def retrieve_serialized_value(
self, value_id: uuid.UUID
) -> Union[SerializedData, None]:
"""Create a LoadConfig object from the details of the persisted version of this value."""
pv = self.retrieve_persisted_value_details(value_id=value_id)
if pv is None:
return None
return pv
retrieve_value_data(self, value, target_profile=None)
¶Source code in kiara/registries/data/__init__.py
def retrieve_value_data(
self, value: Union[uuid.UUID, Value], target_profile: Union[str, None] = None
) -> Any:
if isinstance(value, uuid.UUID):
value = self.get_value(value=value)
if value.value_id in self._cached_data.keys():
return self._cached_data[value.value_id]
if value._serialized_data is None:
serialized_data: Union[
str, SerializedData
] = self.retrieve_persisted_value_details(value_id=value.value_id)
value._serialized_data = serialized_data
else:
serialized_data = value._serialized_data
if isinstance(serialized_data, str):
raise Exception(
f"Can't retrieve serialized version of value '{value.value_id}', this is most likely a bug."
)
manifest = serialized_data.metadata.deserialize.get("python_object", None)
if manifest is None:
raise Exception(
f"No deserialize operation found for data type: {value.data_type_name}"
)
module = self._kiara.create_module(manifest=manifest)
op = Operation.create_from_module(module=module)
input_field_match: Union[str, None] = None
if len(op.inputs_schema) == 1:
input_field_match = next(iter(op.inputs_schema.keys()))
else:
for input_field, schema in op.inputs_schema.items():
for dt in self._kiara.type_registry.get_type_lineage(
value.data_type_name
):
if schema.type == dt:
if input_field_match is not None:
raise Exception(
f"Can't determine input field for deserialization operation '{module.module_type_name}': multiple input fields with type '{input_field_match}'."
)
else:
input_field_match = input_field
break
if input_field_match:
break
if input_field_match is None:
raise Exception(
f"Can't determine input field for deserialization operation '{module.module_type_name}'."
)
result_field_match: Union[str, None] = None
for result_field, schema in op.outputs_schema.items():
if schema.type == "python_object":
if result_field_match is not None:
raise Exception(
f"Can't determine result field for deserialization operation '{module.module_type_name}': multiple result fields with type 'python_object'."
)
else:
result_field_match = result_field
if result_field_match is None:
raise Exception(
f"Can't determine result field for deserialization operation '{module.module_type_name}'."
)
inputs = {input_field_match: value}
result = op.run(kiara=self._kiara, inputs=inputs)
python_object = result.get_value_data(result_field_match)
# TODO: can we do without this?
parsed = value.data_type.parse_python_obj(python_object)
value.data_type._validate(parsed)
self._cached_data[value.value_id] = parsed
return parsed
store_value(self, value, store_id=None)
¶Source code in kiara/registries/data/__init__.py
def store_value(
self,
value: Union[ValueLink, uuid.UUID, str],
store_id: Union[str, None] = None,
) -> Union[PersistedData, None]:
if store_id is None:
store_id = self.default_data_store
_value = self.get_value(value)
store: DataStore = self.get_archive(archive_id=store_id) # type: ignore
if not isinstance(store, DataStore):
raise Exception(f"Can't store value into store '{store_id}': not writable.")
# make sure all property values are available
if _value.pedigree != ORPHAN:
for value_id in _value.pedigree.inputs.values():
self.store_value(value=value_id, store_id=store_id)
if not store.has_value(_value.value_id):
event = ValuePreStoreEvent.construct(kiara_id=self._kiara.id, value=_value)
self._event_callback(event)
persisted_value = store.store_value(_value)
_value._is_stored = True
self._value_archive_lookup_map[_value.value_id] = store_id
self._persisted_value_descs[_value.value_id] = persisted_value
property_values = _value.property_values
for property, property_value in property_values.items():
self.store_value(value=property_value, store_id=store_id)
else:
persisted_value = None
store_event = ValueStoredEvent.construct(kiara_id=self._kiara.id, value=_value)
self._event_callback(store_event)
return persisted_value
DefaultAliasResolver (AliasResolver)
¶Source code in kiara/registries/data/__init__.py
class DefaultAliasResolver(AliasResolver):
def __init__(self, kiara: "Kiara"):
super().__init__(kiara=kiara)
def resolve_alias(self, alias: str) -> uuid.UUID:
if ":" in alias:
ref_type, rest = alias.split(":", maxsplit=1)
if ref_type == "value":
_value_id: Union[uuid.UUID, None] = uuid.UUID(rest)
elif ref_type == "alias":
_value_id = self._kiara.alias_registry.find_value_id_for_alias(
alias=rest
)
if _value_id is None:
raise NoSuchValueAliasException(
alias=rest,
msg=f"Can't retrive value for alias '{rest}': no such alias registered.",
)
else:
raise Exception(
f"Can't retrieve value for '{alias}': invalid reference type '{ref_type}'."
)
else:
_value_id = self._kiara.alias_registry.find_value_id_for_alias(alias)
if _value_id is None:
raise Exception(
f"Can't retrieve value for alias '{alias}': no such alias registered."
)
if _value_id is None:
raise Exception(
f"Can't retrieve value for alias '{alias}': no such alias registered."
)
return _value_id
resolve_alias(self, alias)
¶Source code in kiara/registries/data/__init__.py
def resolve_alias(self, alias: str) -> uuid.UUID:
if ":" in alias:
ref_type, rest = alias.split(":", maxsplit=1)
if ref_type == "value":
_value_id: Union[uuid.UUID, None] = uuid.UUID(rest)
elif ref_type == "alias":
_value_id = self._kiara.alias_registry.find_value_id_for_alias(
alias=rest
)
if _value_id is None:
raise NoSuchValueAliasException(
alias=rest,
msg=f"Can't retrive value for alias '{rest}': no such alias registered.",
)
else:
raise Exception(
f"Can't retrieve value for '{alias}': invalid reference type '{ref_type}'."
)
else:
_value_id = self._kiara.alias_registry.find_value_id_for_alias(alias)
if _value_id is None:
raise Exception(
f"Can't retrieve value for alias '{alias}': no such alias registered."
)
if _value_id is None:
raise Exception(
f"Can't retrieve value for alias '{alias}': no such alias registered."
)
return _value_id
ValueLink (Protocol)
¶Source code in kiara/registries/data/__init__.py
class ValueLink(Protocol):
value_id: uuid.UUID
Modules¶
data_store
special
¶logger
¶
BaseDataStore (DataStore)
¶Source code in kiara/registries/data/data_store/__init__.py
class BaseDataStore(DataStore):
# @abc.abstractmethod
# def _persist_bytes(self, bytes_structure: BytesStructure) -> BytesAliasStructure:
# pass
@abc.abstractmethod
def _persist_stored_value_info(self, value: Value, persisted_value: PersistedData):
pass
@abc.abstractmethod
def _persist_value_details(self, value: Value):
pass
@abc.abstractmethod
def _persist_value_data(self, value: Value) -> PersistedData:
pass
@abc.abstractmethod
def _persist_value_pedigree(self, value: Value):
"""Create an internal link from a value to its pedigree (and pedigree details).
This is so that the 'retrieve_job_record' can be used to prevent running the same job again, and the link of value
to the job that produced it is preserved.
"""
@abc.abstractmethod
def _persist_environment_details(
self, env_type: str, env_hash: str, env_data: Mapping[str, Any]
):
pass
@abc.abstractmethod
def _persist_destiny_backlinks(self, value: Value):
pass
def store_value(self, value: Value) -> PersistedData:
logger.debug(
"store.value",
data_type=value.value_schema.type,
value_id=value.value_id,
value_hash=value.value_hash,
)
# first, persist environment information
for env_type, env_hash in value.pedigree.environments.items():
cached = self._env_cache.get(env_type, {}).get(env_hash, None)
if cached is not None:
continue
env = self.kiara_context.environment_registry.get_environment_for_cid(
env_hash
)
self.persist_environment(env)
# save the value data and metadata
persisted_value = self._persist_value(value)
self._persisted_value_cache[value.value_id] = persisted_value
self._value_cache[value.value_id] = value
self._value_hash_index.setdefault(value.value_hash, set()).add(value.value_id)
# now link the output values to the manifest
# then, make sure the manifest is persisted
self._persist_value_pedigree(value=value)
return persisted_value
def _persist_value(self, value: Value) -> PersistedData:
# TODO: check if value id is already persisted?
if value.is_set:
persisted_value_info: PersistedData = self._persist_value_data(value=value)
if not persisted_value_info:
raise Exception(
"Can't write persisted value info, no load config returned when persisting value."
)
if not isinstance(persisted_value_info, PersistedData):
raise Exception(
f"Can't write persisted value info, invalid result type '{type(persisted_value_info)}' when persisting value."
)
else:
persisted_value_info = PersistedData(
archive_id=self.archive_id,
data_type=value.data_type_name,
serialization_profile="none",
data_type_config=value.data_type_config,
chunk_id_map={},
)
self._persist_stored_value_info(
value=value, persisted_value=persisted_value_info
)
self._persist_value_details(value=value)
if value.destiny_backlinks:
self._persist_destiny_backlinks(value=value)
return persisted_value_info
def persist_environment(self, environment: RuntimeEnvironment):
"""Persist the specified environment.
The environment is stored as a dictionary, including it's schema, not as the actual Python model.
This is to make sure it can still be loaded later on, in case the Python model has changed in later versions.
"""
env_type = environment.get_environment_type_name()
env_hash = str(environment.instance_cid)
env = self._env_cache.get(env_type, {}).get(env_hash, None)
if env is not None:
return
env_data = environment.as_dict_with_schema()
self._persist_environment_details(
env_type=env_type, env_hash=env_hash, env_data=env_data
)
self._env_cache.setdefault(env_type, {})[env_hash] = env_data
def create_renderable(self, **config: Any) -> RenderableType:
"""Create a renderable for this module configuration."""
from kiara.utils.output import create_renderable_from_values
all_values = {}
all_value_ids = self.value_ids
if all_value_ids:
for value_id in all_value_ids:
value = self.kiara_context.data_registry.get_value(value_id)
all_values[str(value_id)] = value
table = create_renderable_from_values(values=all_values, config=config)
return table
else:
return "Data archive does not support statically determined ids."
create_renderable(self, **config)
¶Create a renderable for this module configuration.
Source code in kiara/registries/data/data_store/__init__.py
def create_renderable(self, **config: Any) -> RenderableType:
"""Create a renderable for this module configuration."""
from kiara.utils.output import create_renderable_from_values
all_values = {}
all_value_ids = self.value_ids
if all_value_ids:
for value_id in all_value_ids:
value = self.kiara_context.data_registry.get_value(value_id)
all_values[str(value_id)] = value
table = create_renderable_from_values(values=all_values, config=config)
return table
else:
return "Data archive does not support statically determined ids."
persist_environment(self, environment)
¶Persist the specified environment.
The environment is stored as a dictionary, including it's schema, not as the actual Python model. This is to make sure it can still be loaded later on, in case the Python model has changed in later versions.
Source code in kiara/registries/data/data_store/__init__.py
def persist_environment(self, environment: RuntimeEnvironment):
"""Persist the specified environment.
The environment is stored as a dictionary, including it's schema, not as the actual Python model.
This is to make sure it can still be loaded later on, in case the Python model has changed in later versions.
"""
env_type = environment.get_environment_type_name()
env_hash = str(environment.instance_cid)
env = self._env_cache.get(env_type, {}).get(env_hash, None)
if env is not None:
return
env_data = environment.as_dict_with_schema()
self._persist_environment_details(
env_type=env_type, env_hash=env_hash, env_data=env_data
)
self._env_cache.setdefault(env_type, {})[env_hash] = env_data
store_value(self, value)
¶"Store the value, its data and metadata into the store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Value |
the value to persist |
required |
Returns:
| Type | Description |
|---|---|
PersistedData |
the load config that is needed to retrieve the value data later |
Source code in kiara/registries/data/data_store/__init__.py
def store_value(self, value: Value) -> PersistedData:
logger.debug(
"store.value",
data_type=value.value_schema.type,
value_id=value.value_id,
value_hash=value.value_hash,
)
# first, persist environment information
for env_type, env_hash in value.pedigree.environments.items():
cached = self._env_cache.get(env_type, {}).get(env_hash, None)
if cached is not None:
continue
env = self.kiara_context.environment_registry.get_environment_for_cid(
env_hash
)
self.persist_environment(env)
# save the value data and metadata
persisted_value = self._persist_value(value)
self._persisted_value_cache[value.value_id] = persisted_value
self._value_cache[value.value_id] = value
self._value_hash_index.setdefault(value.value_hash, set()).add(value.value_id)
# now link the output values to the manifest
# then, make sure the manifest is persisted
self._persist_value_pedigree(value=value)
return persisted_value
DataArchive (BaseArchive)
¶Source code in kiara/registries/data/data_store/__init__.py
class DataArchive(BaseArchive):
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["data"]
def __init__(self, archive_id: uuid.UUID, config: ARCHIVE_CONFIG_CLS):
super().__init__(archive_id=archive_id, config=config)
self._env_cache: Dict[str, Dict[str, Mapping[str, Any]]] = {}
self._value_cache: Dict[uuid.UUID, Value] = {}
self._persisted_value_cache: Dict[uuid.UUID, PersistedData] = {}
self._value_hash_index: Dict[str, Set[uuid.UUID]] = {}
def retrieve_serialized_value(
self, value: Union[uuid.UUID, Value]
) -> PersistedData:
if isinstance(value, Value):
value_id: uuid.UUID = value.value_id
_value: Union[Value, None] = value
else:
value_id = value
_value = None
if value_id in self._persisted_value_cache.keys():
return self._persisted_value_cache[value_id]
if _value is None:
_value = self.retrieve_value(value_id)
assert _value is not None
persisted_value = self._retrieve_serialized_value(value=_value)
self._persisted_value_cache[_value.value_id] = persisted_value
return persisted_value
@abc.abstractmethod
def _retrieve_serialized_value(self, value: Value) -> PersistedData:
pass
def retrieve_value(self, value_id: uuid.UUID) -> Value:
cached = self._value_cache.get(value_id, None)
if cached is not None:
return cached
value_data = self._retrieve_value_details(value_id=value_id)
value_schema = ValueSchema(**value_data["value_schema"])
# data_type = self._kiara.get_value_type(
# data_type=value_schema.type, data_type_config=value_schema.type_config
# )
pedigree = ValuePedigree(**value_data["pedigree"])
value = Value(
value_id=value_data["value_id"],
kiara_id=self.kiara_context.id,
value_schema=value_schema,
value_status=value_data["value_status"],
value_size=value_data["value_size"],
value_hash=value_data["value_hash"],
environment_hashes=value_data.get("environment_hashes", {}),
pedigree=pedigree,
pedigree_output_name=value_data["pedigree_output_name"],
data_type_info=value_data["data_type_info"],
property_links=value_data["property_links"],
destiny_backlinks=value_data["destiny_backlinks"],
)
self._value_cache[value_id] = value
return self._value_cache[value_id]
@abc.abstractmethod
def _retrieve_value_details(self, value_id: uuid.UUID) -> Mapping[str, Any]:
pass
@property
def value_ids(self) -> Union[None, Iterable[uuid.UUID]]:
return self._retrieve_all_value_ids()
def _retrieve_all_value_ids(
self, data_type_name: Union[str, None] = None
) -> Union[None, Iterable[uuid.UUID]]:
pass
def has_value(self, value_id: uuid.UUID) -> bool:
"""Check whether the specific value_id is persisted in this data store.
Implementing classes are encouraged to override this method, and choose a suitable, implementation specific
way to quickly determine whether a value id is valid for this data store.
Arguments:
value_id: the id of the value to check.
Returns:
whether this data store contains the value with the specified id
"""
all_value_ids = self.value_ids
if all_value_ids is None:
return False
return value_id in all_value_ids
def retrieve_environment_details(
self, env_type: str, env_hash: str
) -> Mapping[str, Any]:
"""Retrieve the environment details with the specified type and hash.
The environment is stored by the data store as a dictionary, including it's schema, not as the actual Python model.
This is to make sure it can still be loaded later on, in case the Python model has changed in later versions.
"""
cached = self._env_cache.get(env_type, {}).get(env_hash, None)
if cached is not None:
return cached
env = self._retrieve_environment_details(env_type=env_type, env_hash=env_hash)
self._env_cache.setdefault(env_type, {})[env_hash] = env
return env
@abc.abstractmethod
def _retrieve_environment_details(
self, env_type: str, env_hash: str
) -> Mapping[str, Any]:
pass
def find_values(self, matcher: ValueMatcher) -> Iterable[Value]:
raise NotImplementedError()
def find_values_with_hash(
self,
value_hash: str,
value_size: Union[int, None] = None,
data_type_name: Union[str, None] = None,
) -> Set[uuid.UUID]:
if data_type_name is not None:
raise NotImplementedError()
if value_size is not None:
raise NotImplementedError()
if value_hash in self._value_hash_index.keys():
value_ids: Union[Set[uuid.UUID], None] = self._value_hash_index[value_hash]
else:
value_ids = self._find_values_with_hash(
value_hash=value_hash, data_type_name=data_type_name
)
if value_ids is None:
value_ids = set()
self._value_hash_index[value_hash] = value_ids
assert value_ids is not None
return value_ids
@abc.abstractmethod
def _find_values_with_hash(
self,
value_hash: str,
value_size: Union[int, None] = None,
data_type_name: Union[str, None] = None,
) -> Union[Set[uuid.UUID], None]:
pass
def find_destinies_for_value(
self, value_id: uuid.UUID, alias_filter: Union[str, None] = None
) -> Union[Mapping[str, uuid.UUID], None]:
return self._find_destinies_for_value(
value_id=value_id, alias_filter=alias_filter
)
@abc.abstractmethod
def _find_destinies_for_value(
self, value_id: uuid.UUID, alias_filter: Union[str, None] = None
) -> Union[Mapping[str, uuid.UUID], None]:
pass
@abc.abstractmethod
def retrieve_chunk(
self,
chunk_id: str,
as_file: Union[bool, str, None] = None,
symlink_ok: bool = True,
) -> Union[bytes, str]:
pass
# def retrieve_job_record(self, inputs_manifest: InputsManifest) -> Optional[JobRecord]:
# return self._retrieve_job_record(
# manifest_hash=inputs_manifest.manifest_hash, jobs_hash=inputs_manifest.jobs_hash
# )
#
# @abc.abstractmethod
# def _retrieve_job_record(
# self, manifest_hash: int, jobs_hash: int
# ) -> Optional[JobRecord]:
# pass
value_ids: Optional[Iterable[uuid.UUID]]
property
readonly
¶find_destinies_for_value(self, value_id, alias_filter=None)
¶Source code in kiara/registries/data/data_store/__init__.py
def find_destinies_for_value(
self, value_id: uuid.UUID, alias_filter: Union[str, None] = None
) -> Union[Mapping[str, uuid.UUID], None]:
return self._find_destinies_for_value(
value_id=value_id, alias_filter=alias_filter
)
find_values(self, matcher)
¶Source code in kiara/registries/data/data_store/__init__.py
def find_values(self, matcher: ValueMatcher) -> Iterable[Value]:
raise NotImplementedError()
find_values_with_hash(self, value_hash, value_size=None, data_type_name=None)
¶Source code in kiara/registries/data/data_store/__init__.py
def find_values_with_hash(
self,
value_hash: str,
value_size: Union[int, None] = None,
data_type_name: Union[str, None] = None,
) -> Set[uuid.UUID]:
if data_type_name is not None:
raise NotImplementedError()
if value_size is not None:
raise NotImplementedError()
if value_hash in self._value_hash_index.keys():
value_ids: Union[Set[uuid.UUID], None] = self._value_hash_index[value_hash]
else:
value_ids = self._find_values_with_hash(
value_hash=value_hash, data_type_name=data_type_name
)
if value_ids is None:
value_ids = set()
self._value_hash_index[value_hash] = value_ids
assert value_ids is not None
return value_ids
has_value(self, value_id)
¶Check whether the specific value_id is persisted in this data store.
Implementing classes are encouraged to override this method, and choose a suitable, implementation specific way to quickly determine whether a value id is valid for this data store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value_id |
UUID |
the id of the value to check. |
required |
Returns:
| Type | Description |
|---|---|
bool |
whether this data store contains the value with the specified id |
Source code in kiara/registries/data/data_store/__init__.py
def has_value(self, value_id: uuid.UUID) -> bool:
"""Check whether the specific value_id is persisted in this data store.
Implementing classes are encouraged to override this method, and choose a suitable, implementation specific
way to quickly determine whether a value id is valid for this data store.
Arguments:
value_id: the id of the value to check.
Returns:
whether this data store contains the value with the specified id
"""
all_value_ids = self.value_ids
if all_value_ids is None:
return False
return value_id in all_value_ids
retrieve_chunk(self, chunk_id, as_file=None, symlink_ok=True)
¶Source code in kiara/registries/data/data_store/__init__.py
@abc.abstractmethod
def retrieve_chunk(
self,
chunk_id: str,
as_file: Union[bool, str, None] = None,
symlink_ok: bool = True,
) -> Union[bytes, str]:
pass
retrieve_environment_details(self, env_type, env_hash)
¶Retrieve the environment details with the specified type and hash.
The environment is stored by the data store as a dictionary, including it's schema, not as the actual Python model. This is to make sure it can still be loaded later on, in case the Python model has changed in later versions.
Source code in kiara/registries/data/data_store/__init__.py
def retrieve_environment_details(
self, env_type: str, env_hash: str
) -> Mapping[str, Any]:
"""Retrieve the environment details with the specified type and hash.
The environment is stored by the data store as a dictionary, including it's schema, not as the actual Python model.
This is to make sure it can still be loaded later on, in case the Python model has changed in later versions.
"""
cached = self._env_cache.get(env_type, {}).get(env_hash, None)
if cached is not None:
return cached
env = self._retrieve_environment_details(env_type=env_type, env_hash=env_hash)
self._env_cache.setdefault(env_type, {})[env_hash] = env
return env
retrieve_serialized_value(self, value)
¶Source code in kiara/registries/data/data_store/__init__.py
def retrieve_serialized_value(
self, value: Union[uuid.UUID, Value]
) -> PersistedData:
if isinstance(value, Value):
value_id: uuid.UUID = value.value_id
_value: Union[Value, None] = value
else:
value_id = value
_value = None
if value_id in self._persisted_value_cache.keys():
return self._persisted_value_cache[value_id]
if _value is None:
_value = self.retrieve_value(value_id)
assert _value is not None
persisted_value = self._retrieve_serialized_value(value=_value)
self._persisted_value_cache[_value.value_id] = persisted_value
return persisted_value
retrieve_value(self, value_id)
¶Source code in kiara/registries/data/data_store/__init__.py
def retrieve_value(self, value_id: uuid.UUID) -> Value:
cached = self._value_cache.get(value_id, None)
if cached is not None:
return cached
value_data = self._retrieve_value_details(value_id=value_id)
value_schema = ValueSchema(**value_data["value_schema"])
# data_type = self._kiara.get_value_type(
# data_type=value_schema.type, data_type_config=value_schema.type_config
# )
pedigree = ValuePedigree(**value_data["pedigree"])
value = Value(
value_id=value_data["value_id"],
kiara_id=self.kiara_context.id,
value_schema=value_schema,
value_status=value_data["value_status"],
value_size=value_data["value_size"],
value_hash=value_data["value_hash"],
environment_hashes=value_data.get("environment_hashes", {}),
pedigree=pedigree,
pedigree_output_name=value_data["pedigree_output_name"],
data_type_info=value_data["data_type_info"],
property_links=value_data["property_links"],
destiny_backlinks=value_data["destiny_backlinks"],
)
self._value_cache[value_id] = value
return self._value_cache[value_id]
supported_item_types()
classmethod
¶Source code in kiara/registries/data/data_store/__init__.py
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["data"]
DataStore (DataArchive)
¶Source code in kiara/registries/data/data_store/__init__.py
class DataStore(DataArchive):
@classmethod
def is_writeable(cls) -> bool:
return True
@abc.abstractmethod
def store_value(self, value: Value) -> PersistedData:
""" "Store the value, its data and metadata into the store.
Arguments:
value: the value to persist
Returns:
the load config that is needed to retrieve the value data later
"""
is_writeable()
classmethod
¶Source code in kiara/registries/data/data_store/__init__.py
@classmethod
def is_writeable(cls) -> bool:
return True
store_value(self, value)
¶"Store the value, its data and metadata into the store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Value |
the value to persist |
required |
Returns:
| Type | Description |
|---|---|
PersistedData |
the load config that is needed to retrieve the value data later |
Source code in kiara/registries/data/data_store/__init__.py
@abc.abstractmethod
def store_value(self, value: Value) -> PersistedData:
""" "Store the value, its data and metadata into the store.
Arguments:
value: the value to persist
Returns:
the load config that is needed to retrieve the value data later
"""
filesystem_store
¶DEFAULT_HASHFS_DEPTH
¶DEFAULT_HASHFS_WIDTH
¶DEFAULT_HASH_FS_ALGORITHM
¶VALUE_DETAILS_FILE_NAME
¶logger
¶
EntityType (Enum)
¶
FileSystemDataArchive (DataArchive, JobArchive)
¶Data store that loads data from the local filesystem.
Source code in kiara/registries/data/data_store/filesystem_store.py
class FileSystemDataArchive(DataArchive, JobArchive):
"""Data store that loads data from the local filesystem."""
_archive_type_name = "filesystem_data_archive"
_config_cls = FileSystemArchiveConfig
# @classmethod
# def supported_item_types(cls) -> Iterable[str]:
#
# return ["data", "job_record"]
@classmethod
def is_writeable(cls) -> bool:
return False
def __init__(self, archive_id: uuid.UUID, config: FileSystemArchiveConfig):
DataArchive.__init__(self, archive_id=archive_id, config=config)
self._base_path: Union[Path, None] = None
self._hashfs_path: Union[Path, None] = None
self._hashfs: Union[HashFS, None] = None
# def get_job_archive_id(self) -> uuid.UUID:
# return self._kiara.id
def get_archive_details(self) -> ArchiveDetails:
size = sum(
f.stat().st_size for f in self.data_store_path.glob("**/*") if f.is_file()
)
return ArchiveDetails(size=size)
@property
def data_store_path(self) -> Path:
if self._base_path is not None:
return self._base_path
self._base_path = Path(self.config.archive_path).absolute() # type: ignore
self._base_path.mkdir(parents=True, exist_ok=True)
return self._base_path
def _delete_archive(self):
shutil.rmtree(self.data_store_path)
@property
def hash_fs_path(self) -> Path:
if self._hashfs_path is None:
self._hashfs_path = self.data_store_path / "hash_fs"
return self._hashfs_path
@property
def hashfs(self) -> HashFS:
if self._hashfs is None:
self._hashfs = HashFS(
self.hash_fs_path.as_posix(),
depth=DEFAULT_HASHFS_DEPTH,
width=DEFAULT_HASHFS_WIDTH,
algorithm=DEFAULT_HASH_FS_ALGORITHM,
)
return self._hashfs
def get_path(
self,
entity_type: Union[EntityType, None] = None,
base_path: Union[Path, None] = None,
) -> Path:
if base_path is None:
if entity_type is None:
result = self.data_store_path
else:
result = self.data_store_path / entity_type.value
else:
if entity_type is None:
result = base_path
else:
result = base_path / entity_type.value
result.mkdir(parents=True, exist_ok=True)
return result
def _retrieve_environment_details(
self, env_type: str, env_hash: str
) -> Mapping[str, Any]:
base_path = self.get_path(entity_type=EntityType.ENVIRONMENT)
env_details_file = base_path / f"{env_type}_{env_hash}.json"
if not env_details_file.exists():
raise Exception(
f"Can't load environment details, file does not exist: {env_details_file.as_posix()}"
)
environment = orjson.loads(env_details_file.read_text())
return environment
def retrieve_all_job_hashes(
self,
manifest_hash: Union[str, None] = None,
inputs_hash: Union[str, None] = None,
) -> Iterable[str]:
raise NotImplementedError()
def _retrieve_record_for_job_hash(self, job_hash: str) -> JobRecord:
raise NotImplementedError()
# def find_matching_job_record(
# self, inputs_manifest: InputsManifest
# ) -> Optional[JobRecord]:
#
# manifest_hash = str(inputs_manifest.instance_cid)
# jobs_hash = inputs_manifest.job_hash
#
# base_path = self.get_path(entity_type=EntityType.MANIFEST)
# manifest_folder = base_path / str(manifest_hash)
#
# if not manifest_folder.exists():
# return None
#
# manifest_file = manifest_folder / "manifest.json"
#
# if not manifest_file.exists():
# raise Exception(
# f"No 'manifests.json' file for manifest with hash: {manifest_hash}"
# )
#
# manifest_data = orjson.loads(manifest_file.read_text())
#
# job_folder = manifest_folder / jobs_hash
#
# if not job_folder.exists():
# return None
#
# inputs_file_name = job_folder / "inputs.json"
# if not inputs_file_name.exists():
# raise Exception(
# f"No 'inputs.json' file for manifest/inputs hash-combo: {manifest_hash} / {jobs_hash}"
# )
#
# inputs_data = {
# k: uuid.UUID(v)
# for k, v in orjson.loads(inputs_file_name.read_text()).items()
# }
#
# outputs = {}
# for output_file in job_folder.glob("output__*.json"):
# full_output_name = output_file.name[8:]
# start_value_id = full_output_name.find("__value_id__")
# output_name = full_output_name[0:start_value_id]
# value_id_str = full_output_name[start_value_id + 12 : -5] # noqa
#
# value_id = uuid.UUID(value_id_str)
# outputs[output_name] = value_id
#
# job_id = ID_REGISTRY.generate(obj_type=JobRecord, desc="fake job id")
# job_record = JobRecord(
# job_id=job_id,
# module_type=manifest_data["module_type"],
# module_config=manifest_data["module_config"],
# inputs=inputs_data,
# outputs=outputs,
# )
# return job_record
def _find_values_with_hash(
self,
value_hash: str,
value_size: Union[int, None] = None,
data_type_name: Union[str, None] = None,
) -> Set[uuid.UUID]:
value_data_folder = self.get_path(entity_type=EntityType.VALUE_DATA)
glob = f"*/{value_hash}/value_id__*.json"
matches = list(value_data_folder.glob(glob))
result = set()
for match in matches:
if not match.is_symlink():
log_message(
f"Ignoring value_id file, not a symlink: {match.as_posix()}"
)
continue
uuid_str = match.name[10:-5]
value_id = uuid.UUID(uuid_str)
result.add(value_id)
return result
def _find_destinies_for_value(
self, value_id: uuid.UUID, alias_filter: Union[str, None] = None
) -> Union[Mapping[str, uuid.UUID], None]:
destiny_dir = self.get_path(entity_type=EntityType.DESTINY_LINK)
destiny_value_dir = destiny_dir / str(value_id)
if not destiny_value_dir.exists():
return None
destinies = {}
for alias_link in destiny_value_dir.glob("*.json"):
assert alias_link.is_symlink()
alias = alias_link.name[0:-5]
resolved = alias_link.resolve()
value_id_str = resolved.parent.name
value_id = uuid.UUID(value_id_str)
destinies[alias] = value_id
return destinies
def _retrieve_all_value_ids(
self, data_type_name: Union[str, None] = None
) -> Iterable[uuid.UUID]:
if data_type_name is not None:
raise NotImplementedError()
childs = self.get_path(entity_type=EntityType.VALUE).glob("*")
folders = [uuid.UUID(x.name) for x in childs if x.is_dir()]
return folders
def has_value(self, value_id: uuid.UUID) -> bool:
"""Check whether the specific value_id is persisted in this data store.
way to quickly determine whether a value id is valid for this data store.
Arguments:
value_id: the id of the value to check.
Returns:
whether this data store contains the value with the specified id
"""
base_path = (
self.get_path(entity_type=EntityType.VALUE)
/ str(value_id)
/ VALUE_DETAILS_FILE_NAME
)
return base_path.is_file()
def _retrieve_value_details(self, value_id: uuid.UUID) -> Mapping[str, Any]:
base_path = (
self.get_path(entity_type=EntityType.VALUE)
/ str(value_id)
/ VALUE_DETAILS_FILE_NAME
)
if not base_path.is_file():
raise Exception(
f"Can't retrieve details for value with id '{value_id}': no value with that id stored."
)
value_data = orjson.loads(base_path.read_text())
return value_data
def _retrieve_serialized_value(self, value: Value) -> PersistedData:
base_path = self.get_path(entity_type=EntityType.VALUE_DATA)
data_dir = base_path / value.data_type_name / str(value.value_hash)
serialized_value_file = data_dir / ".serialized_value.json"
data = orjson.loads(serialized_value_file.read_text())
return PersistedData(**data)
def retrieve_chunk(
self,
chunk_id: str,
as_file: Union[bool, str, None] = None,
symlink_ok: bool = True,
) -> Union[bytes, str]:
addr = self.hashfs.get(chunk_id)
if as_file in (None, True):
return addr.abspath
elif as_file is False:
return Path(addr.abspath).read_bytes()
else:
raise NotImplementedError()
data_store_path: Path
property
readonly
¶hash_fs_path: Path
property
readonly
¶hashfs: HashFS
property
readonly
¶
_config_cls (ArchiveConfig)
private
pydantic-model
¶Source code in kiara/registries/data/data_store/filesystem_store.py
class FileSystemArchiveConfig(ArchiveConfig):
archive_path: str = Field(
description="The path where the data for this archive is stored."
)
get_archive_details(self)
¶Source code in kiara/registries/data/data_store/filesystem_store.py
def get_archive_details(self) -> ArchiveDetails:
size = sum(
f.stat().st_size for f in self.data_store_path.glob("**/*") if f.is_file()
)
return ArchiveDetails(size=size)
get_path(self, entity_type=None, base_path=None)
¶Source code in kiara/registries/data/data_store/filesystem_store.py
def get_path(
self,
entity_type: Union[EntityType, None] = None,
base_path: Union[Path, None] = None,
) -> Path:
if base_path is None:
if entity_type is None:
result = self.data_store_path
else:
result = self.data_store_path / entity_type.value
else:
if entity_type is None:
result = base_path
else:
result = base_path / entity_type.value
result.mkdir(parents=True, exist_ok=True)
return result
has_value(self, value_id)
¶Check whether the specific value_id is persisted in this data store. way to quickly determine whether a value id is valid for this data store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value_id |
UUID |
the id of the value to check. |
required |
Returns:
| Type | Description |
|---|---|
bool |
whether this data store contains the value with the specified id |
Source code in kiara/registries/data/data_store/filesystem_store.py
def has_value(self, value_id: uuid.UUID) -> bool:
"""Check whether the specific value_id is persisted in this data store.
way to quickly determine whether a value id is valid for this data store.
Arguments:
value_id: the id of the value to check.
Returns:
whether this data store contains the value with the specified id
"""
base_path = (
self.get_path(entity_type=EntityType.VALUE)
/ str(value_id)
/ VALUE_DETAILS_FILE_NAME
)
return base_path.is_file()
is_writeable()
classmethod
¶Source code in kiara/registries/data/data_store/filesystem_store.py
@classmethod
def is_writeable(cls) -> bool:
return False
retrieve_all_job_hashes(self, manifest_hash=None, inputs_hash=None)
¶Retrieve a list of all job record hashes (cids) that match the given filter arguments.
A job record hash includes information about the module type used in the job, the module configuration, as well as input field names and value ids for the values used in those inputs.
If the job archive retrieves its jobs in a dynamic way, this will return 'None'.
Source code in kiara/registries/data/data_store/filesystem_store.py
def retrieve_all_job_hashes(
self,
manifest_hash: Union[str, None] = None,
inputs_hash: Union[str, None] = None,
) -> Iterable[str]:
raise NotImplementedError()
retrieve_chunk(self, chunk_id, as_file=None, symlink_ok=True)
¶Source code in kiara/registries/data/data_store/filesystem_store.py
def retrieve_chunk(
self,
chunk_id: str,
as_file: Union[bool, str, None] = None,
symlink_ok: bool = True,
) -> Union[bytes, str]:
addr = self.hashfs.get(chunk_id)
if as_file in (None, True):
return addr.abspath
elif as_file is False:
return Path(addr.abspath).read_bytes()
else:
raise NotImplementedError()
FilesystemDataStore (FileSystemDataArchive, BaseDataStore)
¶Data store that stores data as files on the local filesystem.
Source code in kiara/registries/data/data_store/filesystem_store.py
class FilesystemDataStore(FileSystemDataArchive, BaseDataStore):
"""Data store that stores data as files on the local filesystem."""
_archive_type_name = "filesystem_data_store"
def _persist_environment_details(
self, env_type: str, env_hash: str, env_data: Mapping[str, Any]
):
base_path = self.get_path(entity_type=EntityType.ENVIRONMENT)
env_details_file = base_path / f"{env_type}_{env_hash}.json"
if not env_details_file.exists():
env_details_file.write_text(orjson_dumps(env_data))
def _persist_stored_value_info(self, value: Value, persisted_value: PersistedData):
working_dir = self.get_path(entity_type=EntityType.VALUE_DATA)
data_dir = working_dir / value.data_type_name / str(value.value_hash)
sv_file = data_dir / ".serialized_value.json"
data_dir.mkdir(exist_ok=True, parents=True)
sv_file.write_text(persisted_value.json())
def _persist_value_details(self, value: Value):
value_dir = self.get_path(entity_type=EntityType.VALUE) / str(value.value_id)
if value_dir.exists():
raise Exception(
f"Can't persist value '{value.value_id}', value directory already exists: {value_dir}"
)
else:
value_dir.mkdir(parents=True, exist_ok=False)
value_file = value_dir / VALUE_DETAILS_FILE_NAME
value_data = value.dict()
value_file.write_text(orjson_dumps(value_data, option=orjson.OPT_NON_STR_KEYS))
def _persist_destiny_backlinks(self, value: Value):
destiny_dir = self.get_path(entity_type=EntityType.DESTINY_LINK)
for value_id, backlink in value.destiny_backlinks.items():
destiny_value_dir = destiny_dir / str(value_id)
destiny_value_dir.mkdir(parents=True, exist_ok=True)
destiny_file = destiny_value_dir / f"{backlink}.json"
assert not destiny_file.exists()
value_dir = self.get_path(entity_type=EntityType.VALUE) / str(
value.value_id
)
value_file = value_dir / VALUE_DETAILS_FILE_NAME
assert value_file.exists()
destiny_file.symlink_to(value_file)
def _persist_value_data(self, value: Value) -> PersistedData:
serialized_value: SerializedData = value.serialized_data
chunk_id_map = {}
for key in serialized_value.get_keys():
data_model = serialized_value.get_serialized_data(key)
if data_model.type == "chunk": # type: ignore
chunks: Iterable[Union[str, BytesIO]] = [BytesIO(data_model.chunk)] # type: ignore
elif data_model.type == "chunks": # type: ignore
chunks = (BytesIO(c) for c in data_model.chunks) # type: ignore
elif data_model.type == "file": # type: ignore
chunks = [data_model.file] # type: ignore
elif data_model.type == "files": # type: ignore
chunks = data_model.files # type: ignore
elif data_model.type == "inline-json": # type: ignore
chunks = [BytesIO(data_model.as_json())] # type: ignore
else:
raise Exception(
f"Invalid serialized data type: {type(data_model)}. Available types: {', '.join(SERIALIZE_TYPES)}"
)
chunk_ids = []
for item in zip(serialized_value.get_cids_for_key(key), chunks):
cid = item[0]
_chunk = item[1]
addr: HashAddress = self.hashfs.put_with_precomputed_hash(
_chunk, str(cid)
)
chunk_ids.append(addr.id)
scids = SerializedChunkIDs(
chunk_id_list=chunk_ids,
archive_id=self.archive_id,
size=data_model.get_size(),
)
scids._data_registry = self.kiara_context.data_registry
chunk_id_map[key] = scids
pers_value = PersistedData(
archive_id=self.archive_id,
chunk_id_map=chunk_id_map,
data_type=serialized_value.data_type,
data_type_config=serialized_value.data_type_config,
serialization_profile=serialized_value.serialization_profile,
metadata=serialized_value.metadata,
)
return pers_value
def _persist_value_pedigree(self, value: Value):
manifest_hash = value.pedigree.instance_cid
jobs_hash = value.pedigree.job_hash
base_path = self.get_path(entity_type=EntityType.MANIFEST)
manifest_folder = base_path / str(manifest_hash)
manifest_folder.mkdir(parents=True, exist_ok=True)
manifest_info_file = manifest_folder / "manifest.json"
if not manifest_info_file.exists():
manifest_info_file.write_text(value.pedigree.manifest_data_as_json())
job_folder = manifest_folder / str(jobs_hash)
job_folder.mkdir(parents=True, exist_ok=True)
inputs_details_file_name = job_folder / "inputs.json"
if not inputs_details_file_name.exists():
inputs_details_file_name.write_text(orjson_dumps(value.pedigree.inputs))
outputs_file_name = (
job_folder
/ f"output__{value.pedigree_output_name}__value_id__{value.value_id}.json"
)
if outputs_file_name.exists():
# if value.pedigree_output_name == "__void__":
# return
# else:
raise Exception(f"Can't write value '{value.value_id}': already exists.")
else:
outputs_file_name.touch()
value_data_dir = (
self.get_path(entity_type=EntityType.VALUE_DATA)
/ value.value_schema.type
/ str(value.value_hash)
)
target_file = value_data_dir / f"value_id__{value.value_id}.json"
target_file.symlink_to(outputs_file_name)
destinies
special
¶
Classes¶
DestinyArchive (BaseArchive)
¶Source code in kiara/registries/destinies/__init__.py
class DestinyArchive(BaseArchive):
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["destiny"]
def __init__(self, archive_id: uuid.UUID, config: ARCHIVE_CONFIG_CLS):
super().__init__(archive_id=archive_id, config=config)
@abc.abstractmethod
def get_all_value_ids(self) -> Set[uuid.UUID]:
"""Retrun a list of all value ids that have destinies stored in this archive."""
@abc.abstractmethod
def get_destiny_aliases_for_value(
self, value_id: uuid.UUID
) -> Union[Set[str], None]:
"""Retrieve all the destinies for the specified value within this archive.
In case this archive discovers its value destinies dynamically, this can return 'None'.
"""
@abc.abstractmethod
def get_destiny(self, value_id: uuid.UUID, destiny: str) -> Destiny:
pass
get_all_value_ids(self)
¶Retrun a list of all value ids that have destinies stored in this archive.
Source code in kiara/registries/destinies/__init__.py
@abc.abstractmethod
def get_all_value_ids(self) -> Set[uuid.UUID]:
"""Retrun a list of all value ids that have destinies stored in this archive."""
get_destiny(self, value_id, destiny)
¶Source code in kiara/registries/destinies/__init__.py
@abc.abstractmethod
def get_destiny(self, value_id: uuid.UUID, destiny: str) -> Destiny:
pass
get_destiny_aliases_for_value(self, value_id)
¶Retrieve all the destinies for the specified value within this archive.
In case this archive discovers its value destinies dynamically, this can return 'None'.
Source code in kiara/registries/destinies/__init__.py
@abc.abstractmethod
def get_destiny_aliases_for_value(
self, value_id: uuid.UUID
) -> Union[Set[str], None]:
"""Retrieve all the destinies for the specified value within this archive.
In case this archive discovers its value destinies dynamically, this can return 'None'.
"""
supported_item_types()
classmethod
¶Source code in kiara/registries/destinies/__init__.py
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["destiny"]
DestinyStore (DestinyArchive)
¶Source code in kiara/registries/destinies/__init__.py
class DestinyStore(DestinyArchive):
@abc.abstractmethod
def persist_destiny(self, destiny: Destiny):
pass
persist_destiny(self, destiny)
¶Source code in kiara/registries/destinies/__init__.py
@abc.abstractmethod
def persist_destiny(self, destiny: Destiny):
pass
Modules¶
filesystem_store
¶logger
¶
FileSystemDestinyArchive (DestinyArchive)
¶Source code in kiara/registries/destinies/filesystem_store.py
class FileSystemDestinyArchive(DestinyArchive):
_archive_type_name = "filesystem_destiny_archive"
_config_cls = FileSystemArchiveConfig
@classmethod
def is_writeable(cls) -> bool:
return False
# @classmethod
# def create_from_kiara_context(cls, kiara: "Kiara"):
#
# TODO = kiara_app_dirs.user_data_dir
# base_path = Path(TODO) / "destiny_store"
# base_path.mkdir(parents=True, exist_ok=True)
# result = cls(base_path=base_path, store_id=kiara.id)
# ID_REGISTRY.update_metadata(
# result.get_destiny_archive_id(), kiara_id=kiara.id, obj=result
# )
# return result
def __init__(self, archive_id: uuid.UUID, config: FileSystemArchiveConfig):
super().__init__(archive_id=archive_id, config=config)
self._base_path: Union[Path, None] = None
# base_path = config.archive_path
# if not base_path.is_dir():
# raise Exception(
# f"Can't create file system archive instance, base path does not exist or is not a folder: {base_path.as_posix()}."
# )
# self._store_id: uuid.UUID = store_id
# self._base_path: Path = base_path
# self._destinies_path: Path = self._base_path / "destinies"
# self._value_id_path: Path = self._base_path / "value_ids"
@property
def destiny_store_path(self) -> Path:
if self._base_path is not None:
return self._base_path
self._base_path = Path(self.config.archive_path).absolute() # type: ignore
self._base_path.mkdir(parents=True, exist_ok=True)
return self._base_path
def get_archive_details(self) -> ArchiveDetails:
size = sum(
f.stat().st_size
for f in self.destiny_store_path.glob("**/*")
if f.is_file()
)
return ArchiveDetails(size=size)
@property
def destinies_path(self) -> Path:
return self.destiny_store_path / "destinies"
@property
def value_id_path(self) -> Path:
return self.destiny_store_path / "value_ids"
def _translate_destiny_id_to_path(self, destiny_id: uuid.UUID) -> Path:
tokens = str(destiny_id).split("-")
destiny_path = (
self.destinies_path.joinpath(*tokens[0:-1]) / f"{tokens[-1]}.json"
)
return destiny_path
def _translate_destinies_path_to_id(self, destinies_path: Path) -> uuid.UUID:
relative = destinies_path.relative_to(self.destinies_path).as_posix()[:-5]
destninies_id = "-".join(relative.split(os.path.sep))
return uuid.UUID(destninies_id)
def _translate_value_id(self, value_id: uuid.UUID, destiny_alias: str) -> Path:
tokens = str(value_id).split("-")
value_id_path = self.value_id_path.joinpath(*tokens)
full_path = value_id_path / f"{destiny_alias}.json"
return full_path
def _translate_value_id_path(self, value_path: Path) -> uuid.UUID:
relative = value_path.relative_to(self.value_id_path)
value_id_str = "-".join(relative.as_posix().split(os.path.sep))
return uuid.UUID(value_id_str)
def _translate_alias_path(self, alias_path: Path) -> Tuple[uuid.UUID, str]:
value_id = self._translate_value_id_path(alias_path.parent)
alias = alias_path.name[0:-5]
return value_id, alias
def get_all_value_ids(self) -> Set[uuid.UUID]:
all_root_folders = self.value_id_path.glob("*/*/*/*/*")
result = set()
for folder in all_root_folders:
if not folder.is_dir():
continue
value_id = self._translate_value_id_path(folder)
result.add(value_id)
return result
def get_destiny_aliases_for_value(self, value_id: uuid.UUID) -> Set[str]:
tokens = str(value_id).split("-")
value_id_path = self.value_id_path.joinpath(*tokens)
aliases = value_id_path.glob("*.json")
return set(a.name[0:-5] for a in aliases)
def get_destiny(self, value_id: uuid.UUID, destiny_alias: str) -> Destiny:
tokens = str(value_id).split("-")
value_id_path = self.value_id_path.joinpath(*tokens)
destiny_path = value_id_path / f"{destiny_alias}.json"
destiny_data = orjson.loads(destiny_path.read_text())
destiny = Destiny.construct(**destiny_data)
return destiny
destinies_path: Path
property
readonly
¶destiny_store_path: Path
property
readonly
¶value_id_path: Path
property
readonly
¶
_config_cls (ArchiveConfig)
private
pydantic-model
¶Source code in kiara/registries/destinies/filesystem_store.py
class FileSystemArchiveConfig(ArchiveConfig):
archive_path: str = Field(
description="The path where the data for this archive is stored."
)
get_all_value_ids(self)
¶Retrun a list of all value ids that have destinies stored in this archive.
Source code in kiara/registries/destinies/filesystem_store.py
def get_all_value_ids(self) -> Set[uuid.UUID]:
all_root_folders = self.value_id_path.glob("*/*/*/*/*")
result = set()
for folder in all_root_folders:
if not folder.is_dir():
continue
value_id = self._translate_value_id_path(folder)
result.add(value_id)
return result
get_archive_details(self)
¶Source code in kiara/registries/destinies/filesystem_store.py
def get_archive_details(self) -> ArchiveDetails:
size = sum(
f.stat().st_size
for f in self.destiny_store_path.glob("**/*")
if f.is_file()
)
return ArchiveDetails(size=size)
get_destiny(self, value_id, destiny_alias)
¶Source code in kiara/registries/destinies/filesystem_store.py
def get_destiny(self, value_id: uuid.UUID, destiny_alias: str) -> Destiny:
tokens = str(value_id).split("-")
value_id_path = self.value_id_path.joinpath(*tokens)
destiny_path = value_id_path / f"{destiny_alias}.json"
destiny_data = orjson.loads(destiny_path.read_text())
destiny = Destiny.construct(**destiny_data)
return destiny
get_destiny_aliases_for_value(self, value_id)
¶Retrieve all the destinies for the specified value within this archive.
In case this archive discovers its value destinies dynamically, this can return 'None'.
Source code in kiara/registries/destinies/filesystem_store.py
def get_destiny_aliases_for_value(self, value_id: uuid.UUID) -> Set[str]:
tokens = str(value_id).split("-")
value_id_path = self.value_id_path.joinpath(*tokens)
aliases = value_id_path.glob("*.json")
return set(a.name[0:-5] for a in aliases)
is_writeable()
classmethod
¶Source code in kiara/registries/destinies/filesystem_store.py
@classmethod
def is_writeable(cls) -> bool:
return False
FileSystemDestinyStore (FileSystemDestinyArchive, DestinyStore)
¶Source code in kiara/registries/destinies/filesystem_store.py
class FileSystemDestinyStore(FileSystemDestinyArchive, DestinyStore):
_archive_type_name = "filesystem_destiny_store"
@classmethod
def is_writeable(cls) -> bool:
return True
def persist_destiny(self, destiny: Destiny):
destiny_path = self._translate_destiny_id_to_path(destiny_id=destiny.destiny_id)
destiny_path.parent.mkdir(parents=True, exist_ok=True)
destiny_path.write_text(destiny.json())
for value_id in destiny.fixed_inputs.values():
path = self._translate_value_id(
value_id=value_id, destiny_alias=destiny.destiny_alias
)
if path.exists():
logger.debug("replace.destiny.file", path=path.as_posix())
path.unlink()
# raise Exception(
# f"Can't persist destiny '{destiny.destiny_id}': already persisted."
# )
path.parent.mkdir(parents=True, exist_ok=True)
path.symlink_to(destiny_path)
is_writeable()
classmethod
¶Source code in kiara/registries/destinies/filesystem_store.py
@classmethod
def is_writeable(cls) -> bool:
return True
persist_destiny(self, destiny)
¶Source code in kiara/registries/destinies/filesystem_store.py
def persist_destiny(self, destiny: Destiny):
destiny_path = self._translate_destiny_id_to_path(destiny_id=destiny.destiny_id)
destiny_path.parent.mkdir(parents=True, exist_ok=True)
destiny_path.write_text(destiny.json())
for value_id in destiny.fixed_inputs.values():
path = self._translate_value_id(
value_id=value_id, destiny_alias=destiny.destiny_alias
)
if path.exists():
logger.debug("replace.destiny.file", path=path.as_posix())
path.unlink()
# raise Exception(
# f"Can't persist destiny '{destiny.destiny_id}': already persisted."
# )
path.parent.mkdir(parents=True, exist_ok=True)
path.symlink_to(destiny_path)
registry
¶
DestinyRegistry
¶Source code in kiara/registries/destinies/registry.py
class DestinyRegistry(object):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._event_callback: Callable = self._kiara.event_registry.add_producer(self)
self._destiny_archives: Dict[str, DestinyArchive] = {}
self._default_destiny_store: Union[str, None] = None
# default_metadata_archive = FileSystemDestinyStore.create_from_kiara_context(
# self._kiara
# )
# self.register_destiny_archive("metadata", default_metadata_archive)
self._all_values: Union[Dict[uuid.UUID, Set[str]], None] = None
self._cached_value_aliases: Dict[
uuid.UUID, Dict[str, Union[Destiny, None]]
] = {}
self._destinies: Dict[uuid.UUID, Destiny] = {}
self._destinies_by_value: Dict[uuid.UUID, Dict[str, Destiny]] = {}
self._destiny_store_map: Dict[uuid.UUID, str] = {}
@property
def default_destiny_store(self) -> DestinyStore:
if self._default_destiny_store is None:
raise Exception("No default destiny store set (yet).")
return self._destiny_archives[self._default_destiny_store] # type: ignore
@property
def destiny_archives(self) -> Mapping[str, DestinyArchive]:
return self._destiny_archives
def register_destiny_archive(
self,
archive: DestinyArchive,
alias: str = None,
set_as_default_store: Union[bool, None] = None,
):
destiny_store_id = archive.archive_id
archive.register_archive(kiara=self._kiara)
if alias is None:
alias = str(destiny_store_id)
if alias in self._destiny_archives.keys():
raise Exception(
f"Can't add destiny archive, alias '{alias}' already registered."
)
self._destiny_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, DestinyStore):
is_store = True
if set_as_default_store and self._default_destiny_store is not None:
raise Exception(
f"Can't set data store '{alias}' as default store: default store already set."
)
if self._default_destiny_store is None or set_as_default_store:
is_default_store = True
self._default_destiny_store = alias
event = DestinyArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
destiny_archive_id=archive.archive_id,
destiny_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
# if not registered_name.isalnum():
# raise Exception(
# f"Can't register destiny archive with name '{registered_name}: name must only contain alphanumeric characters.'"
# )
#
# if registered_name in self._destiny_archives.keys():
# raise Exception(
# f"Can't register alias store, store id already registered: {registered_name}."
# )
#
# self._destiny_archives[registered_name] = alias_store
#
# if self._default_destiny_store is None and isinstance(
# alias_store, DestinyStore
# ):
# self._default_destiny_store = registered_name
def _extract_archive(self, alias: str) -> Tuple[str, str]:
if "." not in alias:
assert self._default_destiny_store is not None
return (self._default_destiny_store, alias)
store_id, rest = alias.split(".", maxsplit=1)
if store_id not in self._destiny_archives.keys():
assert self._default_destiny_store is not None
return (self._default_destiny_store, alias)
else:
return (store_id, rest)
def add_destiny(
self,
destiny_alias: str,
values: Dict[str, uuid.UUID],
manifest: Manifest,
result_field_name: Union[str, None] = None,
) -> Destiny:
"""Add a destiny for one (or in some rare cases several) values.
A destiny alias must be unique for every one of the involved input values.
"""
if not values:
raise Exception("Can't add destiny, no values provided.")
store_id, alias = self._extract_archive(destiny_alias)
destiny = Destiny.create_from_values(
kiara=self._kiara,
destiny_alias=alias,
manifest=manifest,
result_field_name=result_field_name,
values=values,
)
for value_id in destiny.fixed_inputs.values():
self._destinies[destiny.destiny_id] = destiny
# TODO: store history?
self._destinies_by_value.setdefault(value_id, {})[destiny_alias] = destiny
self._cached_value_aliases.setdefault(value_id, {})[destiny_alias] = destiny
self._destiny_store_map[destiny.destiny_id] = store_id
return destiny
def get_destiny(self, value_id: uuid.UUID, destiny_alias: str) -> Destiny:
destiny = self._destinies_by_value.get(value_id, {}).get(destiny_alias, None)
if destiny is None:
raise Exception(
f"No destiny '{destiny_alias}' available for value '{value_id}'."
)
return destiny
@property
def _all_values_store_map(self) -> Dict[uuid.UUID, Set[str]]:
if self._all_values is not None:
return self._all_values
all_values: Dict[uuid.UUID, Set[str]] = {}
for archive_id, archive in self._destiny_archives.items():
all_value_ids = archive.get_all_value_ids()
for v_id in all_value_ids:
all_values.setdefault(v_id, set()).add(archive_id)
self._all_values = all_values
return self._all_values
@property
def all_values(self) -> Iterable[uuid.UUID]:
all_stored_values = set(self._all_values_store_map.keys())
all_stored_values.update(self._destinies_by_value.keys())
return all_stored_values
def get_destiny_aliases_for_value(
self, value_id: uuid.UUID, alias_filter: Union[str, None] = None
) -> Iterable[str]:
# TODO: cache the result of this
if alias_filter is not None:
raise NotImplementedError()
all_stores = self._all_values_store_map.get(value_id)
aliases: Set[str] = set()
if all_stores:
for prefix in all_stores:
all_aliases = self._destiny_archives[
prefix
].get_destiny_aliases_for_value(value_id=value_id)
if all_aliases is not None:
aliases.update((f"{prefix}.{a}" for a in all_aliases))
current = self._destinies_by_value.get(value_id, None)
if current:
aliases.update(current.keys())
return sorted(aliases)
# def get_destinies_for_value(
# self,
# value_id: uuid.UUID,
# destiny_alias_filter: Optional[str] = None
# ) -> Mapping[str, Destiny]:
#
#
#
# return self._destinies_by_value.get(value_id, {})
def resolve_destiny(self, destiny: Destiny) -> Value:
results = self._kiara.job_registry.execute_and_retrieve(
manifest=destiny, inputs=destiny.merged_inputs
)
value = results.get_value_obj(field_name=destiny.result_field_name)
destiny.result_value_id = value.value_id
return value
def attach_as_property(
self,
destiny: Union[uuid.UUID, Destiny],
field_names: Union[Iterable[str], None] = None,
):
if field_names:
raise NotImplementedError()
if isinstance(destiny, uuid.UUID):
destiny = self._destinies[destiny]
values = self._kiara.data_registry.load_values(destiny.fixed_inputs)
already_stored: List[uuid.UUID] = []
for v in values.values():
if v.is_stored:
already_stored.append(v.value_id)
if already_stored:
stored = (str(v) for v in already_stored)
raise Exception(
f"Can't attach destiny as property, value(s) already stored: {', '.join(stored)}"
)
store_id = self._destiny_store_map[destiny.destiny_id]
full_path = f"{store_id}.{destiny.destiny_alias}"
for v in values.values():
assert destiny.result_value_id is not None
v.add_property(
value_id=destiny.result_value_id,
property_path=full_path,
add_origin_to_property_value=True,
)
def store_destiny(self, destiny_id: Union[Destiny, uuid.UUID]):
try:
_destiny_id: uuid.UUID = destiny_id.destiny_id # type: ignore
except Exception:
# just in case this is a 'Destiny' object
_destiny_id = destiny_id # type: ignore
store_id = self._destiny_store_map[_destiny_id]
destiny = self._destinies[_destiny_id]
store: DestinyStore = self._destiny_archives[store_id] # type: ignore
if not isinstance(store, DestinyStore):
full_alias = f"{store_id}.{destiny.destiny_alias}"
raise Exception(
f"Can't store destiny '{full_alias}': prefix '{store_id}' not writable in this kiara context."
)
store.persist_destiny(destiny=destiny)
all_values: Iterable[uuid.UUID]
property
readonly
¶default_destiny_store: DestinyStore
property
readonly
¶destiny_archives: Mapping[str, kiara.registries.destinies.DestinyArchive]
property
readonly
¶add_destiny(self, destiny_alias, values, manifest, result_field_name=None)
¶Add a destiny for one (or in some rare cases several) values.
A destiny alias must be unique for every one of the involved input values.
Source code in kiara/registries/destinies/registry.py
def add_destiny(
self,
destiny_alias: str,
values: Dict[str, uuid.UUID],
manifest: Manifest,
result_field_name: Union[str, None] = None,
) -> Destiny:
"""Add a destiny for one (or in some rare cases several) values.
A destiny alias must be unique for every one of the involved input values.
"""
if not values:
raise Exception("Can't add destiny, no values provided.")
store_id, alias = self._extract_archive(destiny_alias)
destiny = Destiny.create_from_values(
kiara=self._kiara,
destiny_alias=alias,
manifest=manifest,
result_field_name=result_field_name,
values=values,
)
for value_id in destiny.fixed_inputs.values():
self._destinies[destiny.destiny_id] = destiny
# TODO: store history?
self._destinies_by_value.setdefault(value_id, {})[destiny_alias] = destiny
self._cached_value_aliases.setdefault(value_id, {})[destiny_alias] = destiny
self._destiny_store_map[destiny.destiny_id] = store_id
return destiny
attach_as_property(self, destiny, field_names=None)
¶Source code in kiara/registries/destinies/registry.py
def attach_as_property(
self,
destiny: Union[uuid.UUID, Destiny],
field_names: Union[Iterable[str], None] = None,
):
if field_names:
raise NotImplementedError()
if isinstance(destiny, uuid.UUID):
destiny = self._destinies[destiny]
values = self._kiara.data_registry.load_values(destiny.fixed_inputs)
already_stored: List[uuid.UUID] = []
for v in values.values():
if v.is_stored:
already_stored.append(v.value_id)
if already_stored:
stored = (str(v) for v in already_stored)
raise Exception(
f"Can't attach destiny as property, value(s) already stored: {', '.join(stored)}"
)
store_id = self._destiny_store_map[destiny.destiny_id]
full_path = f"{store_id}.{destiny.destiny_alias}"
for v in values.values():
assert destiny.result_value_id is not None
v.add_property(
value_id=destiny.result_value_id,
property_path=full_path,
add_origin_to_property_value=True,
)
get_destiny(self, value_id, destiny_alias)
¶Source code in kiara/registries/destinies/registry.py
def get_destiny(self, value_id: uuid.UUID, destiny_alias: str) -> Destiny:
destiny = self._destinies_by_value.get(value_id, {}).get(destiny_alias, None)
if destiny is None:
raise Exception(
f"No destiny '{destiny_alias}' available for value '{value_id}'."
)
return destiny
get_destiny_aliases_for_value(self, value_id, alias_filter=None)
¶Source code in kiara/registries/destinies/registry.py
def get_destiny_aliases_for_value(
self, value_id: uuid.UUID, alias_filter: Union[str, None] = None
) -> Iterable[str]:
# TODO: cache the result of this
if alias_filter is not None:
raise NotImplementedError()
all_stores = self._all_values_store_map.get(value_id)
aliases: Set[str] = set()
if all_stores:
for prefix in all_stores:
all_aliases = self._destiny_archives[
prefix
].get_destiny_aliases_for_value(value_id=value_id)
if all_aliases is not None:
aliases.update((f"{prefix}.{a}" for a in all_aliases))
current = self._destinies_by_value.get(value_id, None)
if current:
aliases.update(current.keys())
return sorted(aliases)
register_destiny_archive(self, archive, alias=None, set_as_default_store=None)
¶Source code in kiara/registries/destinies/registry.py
def register_destiny_archive(
self,
archive: DestinyArchive,
alias: str = None,
set_as_default_store: Union[bool, None] = None,
):
destiny_store_id = archive.archive_id
archive.register_archive(kiara=self._kiara)
if alias is None:
alias = str(destiny_store_id)
if alias in self._destiny_archives.keys():
raise Exception(
f"Can't add destiny archive, alias '{alias}' already registered."
)
self._destiny_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, DestinyStore):
is_store = True
if set_as_default_store and self._default_destiny_store is not None:
raise Exception(
f"Can't set data store '{alias}' as default store: default store already set."
)
if self._default_destiny_store is None or set_as_default_store:
is_default_store = True
self._default_destiny_store = alias
event = DestinyArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
destiny_archive_id=archive.archive_id,
destiny_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
# if not registered_name.isalnum():
# raise Exception(
# f"Can't register destiny archive with name '{registered_name}: name must only contain alphanumeric characters.'"
# )
#
# if registered_name in self._destiny_archives.keys():
# raise Exception(
# f"Can't register alias store, store id already registered: {registered_name}."
# )
#
# self._destiny_archives[registered_name] = alias_store
#
# if self._default_destiny_store is None and isinstance(
# alias_store, DestinyStore
# ):
# self._default_destiny_store = registered_name
resolve_destiny(self, destiny)
¶Source code in kiara/registries/destinies/registry.py
def resolve_destiny(self, destiny: Destiny) -> Value:
results = self._kiara.job_registry.execute_and_retrieve(
manifest=destiny, inputs=destiny.merged_inputs
)
value = results.get_value_obj(field_name=destiny.result_field_name)
destiny.result_value_id = value.value_id
return value
store_destiny(self, destiny_id)
¶Source code in kiara/registries/destinies/registry.py
def store_destiny(self, destiny_id: Union[Destiny, uuid.UUID]):
try:
_destiny_id: uuid.UUID = destiny_id.destiny_id # type: ignore
except Exception:
# just in case this is a 'Destiny' object
_destiny_id = destiny_id # type: ignore
store_id = self._destiny_store_map[_destiny_id]
destiny = self._destinies[_destiny_id]
store: DestinyStore = self._destiny_archives[store_id] # type: ignore
if not isinstance(store, DestinyStore):
full_alias = f"{store_id}.{destiny.destiny_alias}"
raise Exception(
f"Can't store destiny '{full_alias}': prefix '{store_id}' not writable in this kiara context."
)
store.persist_destiny(destiny=destiny)
environment
special
¶
Classes¶
EnvironmentRegistry
¶Source code in kiara/registries/environment/__init__.py
class EnvironmentRegistry(object):
_instance = None
@classmethod
def instance(cls):
"""The default *kiara* context. In most cases, it's recommended you create and manage your own, though."""
if cls._instance is None:
cls._instance = EnvironmentRegistry()
return cls._instance
def __init__(
self,
):
self._environments: Union[Dict[str, RuntimeEnvironment], None] = None
self._environment_hashes: Union[Dict[str, Mapping[str, str]], None] = None
self._full_env_model: Union[BaseModel, None] = None
def get_environment_for_cid(self, env_cid: str) -> RuntimeEnvironment:
envs = [env for env in self.environments.values() if env.instance_id == env_cid]
if len(envs) == 0:
raise Exception(f"No environment with id '{env_cid}' available.")
elif len(envs) > 1:
raise Exception(
f"Multipe environments with id '{env_cid}' available. This is most likely a bug."
)
return envs[0]
@property
def environment_hashes(self) -> Mapping[str, Mapping[str, str]]:
if self._environment_hashes is not None:
return self._environment_hashes
result = {}
for env_name, env in self.environments.items():
result[env_name] = env.env_hashes
self._environment_hashes = result
return self._environment_hashes
@property
def environments(self) -> Mapping[str, RuntimeEnvironment]:
"""Return all environments in this kiara runtime context."""
if self._environments is not None:
return self._environments
import kiara.models.runtime_environment.kiara # noqa
import kiara.models.runtime_environment.operating_system # noqa
import kiara.models.runtime_environment.python # noqa
subclasses: Iterable[Type[RuntimeEnvironment]] = _get_all_subclasses(
RuntimeEnvironment # type: ignore
)
envs = {}
for sc in subclasses:
if inspect.isabstract(sc):
if is_debug():
logger.warning("class_loading.ignore_subclass", subclass=sc)
else:
logger.debug("class_loading.ignore_subclass", subclass=sc)
name = sc.get_environment_type_name()
envs[name] = sc.create_environment_model()
self._environments = {k: envs[k] for k in sorted(envs.keys())}
return self._environments
@property
def full_model(self) -> BaseModel:
"""A model containing all environment data, incl. schemas and hashes of each sub-environment."""
if self._full_env_model is not None:
return self._full_env_model
attrs = {k: (v.__class__, ...) for k, v in self.environments.items()}
models = {}
hashes = {}
schemas = {}
for k, v in attrs.items():
name = to_camel_case(f"{k}_environment")
k_cls: Type[RuntimeEnvironment] = create_model(
name,
__base__=v[0],
metadata_hash=(
str,
Field(
description="The hash for this metadata (excl. this and the 'metadata_schema' field)."
),
),
metadata_schema=(
str,
Field(
description="JsonSchema describing this metadata (excl. this and the 'metadata_hash' field)."
),
),
)
models[k] = (
k_cls,
Field(description=f"Metadata describing the {k} environment."),
)
schemas[k] = v[0].schema_json()
hashes[k] = self.environments[k].instance_cid
cls: Type[BaseModel] = create_model("KiaraRuntimeInfo", **models) # type: ignore
data = {}
for k2, v2 in self.environments.items():
d = v2.dict()
assert "metadata_hash" not in d.keys()
assert "metadata_schema" not in d.keys()
d["metadata_hash"] = str(hashes[k2])
d["metadata_schema"] = schemas[k]
data[k2] = d
model = cls.construct(**data) # type: ignore
self._full_env_model = model
return self._full_env_model
def create_renderable(self, **config: Any):
full_details = config.get("full_details", False)
table = Table(show_header=True, box=box.SIMPLE)
table.add_column("environment key", style="b")
table.add_column("details")
for env_name, env in self.environments.items():
renderable = env.create_renderable(summary=not full_details)
table.add_row(env_name, renderable)
return table
environment_hashes: Mapping[str, Mapping[str, str]]
property
readonly
¶environments: Mapping[str, kiara.models.runtime_environment.RuntimeEnvironment]
property
readonly
¶Return all environments in this kiara runtime context.
full_model: BaseModel
property
readonly
¶A model containing all environment data, incl. schemas and hashes of each sub-environment.
create_renderable(self, **config)
¶Source code in kiara/registries/environment/__init__.py
def create_renderable(self, **config: Any):
full_details = config.get("full_details", False)
table = Table(show_header=True, box=box.SIMPLE)
table.add_column("environment key", style="b")
table.add_column("details")
for env_name, env in self.environments.items():
renderable = env.create_renderable(summary=not full_details)
table.add_row(env_name, renderable)
return table
get_environment_for_cid(self, env_cid)
¶Source code in kiara/registries/environment/__init__.py
def get_environment_for_cid(self, env_cid: str) -> RuntimeEnvironment:
envs = [env for env in self.environments.values() if env.instance_id == env_cid]
if len(envs) == 0:
raise Exception(f"No environment with id '{env_cid}' available.")
elif len(envs) > 1:
raise Exception(
f"Multipe environments with id '{env_cid}' available. This is most likely a bug."
)
return envs[0]
instance()
classmethod
¶The default kiara context. In most cases, it's recommended you create and manage your own, though.
Source code in kiara/registries/environment/__init__.py
@classmethod
def instance(cls):
"""The default *kiara* context. In most cases, it's recommended you create and manage your own, though."""
if cls._instance is None:
cls._instance = EnvironmentRegistry()
return cls._instance
events
special
¶
AsyncEventListener (Protocol)
¶Source code in kiara/registries/events/__init__.py
class AsyncEventListener(Protocol):
def wait_for_processing(self, processing_id: Any):
pass
wait_for_processing(self, processing_id)
¶Source code in kiara/registries/events/__init__.py
def wait_for_processing(self, processing_id: Any):
pass
EventListener (Protocol)
¶Source code in kiara/registries/events/__init__.py
class EventListener(Protocol):
def handle_events(self, *events: KiaraEvent) -> Any:
pass
handle_events(self, *events)
¶Source code in kiara/registries/events/__init__.py
def handle_events(self, *events: KiaraEvent) -> Any:
pass
EventProducer (Protocol)
¶Source code in kiara/registries/events/__init__.py
class EventProducer(Protocol):
pass
# def suppoerted_event_types(self) -> Iterable[Type[KiaraEvent]]:
# pass
metadata
¶
CreateMetadataDestinies
¶Source code in kiara/registries/events/metadata.py
class CreateMetadataDestinies(object):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._skip_internal_types: bool = True
def supported_event_types(self) -> Iterable[str]:
return ["value_created", "value_registered"]
def handle_events(self, *events: KiaraEvent) -> Any:
for event in events:
if event.get_event_type() == "value_created": # type: ignore
if event.value.is_set: # type: ignore
self.attach_metadata(event.value) # type: ignore
for event in events:
if event.get_event_type() == "value_registered": # type: ignore
self.resolve_all_metadata(event.value) # type: ignore
def attach_metadata(self, value: Value):
assert not value.is_stored
if self._skip_internal_types:
if value.value_schema.type == "any":
return
lineage = self._kiara.type_registry.get_type_lineage(
value.value_schema.type
)
if "any" not in lineage:
return
op_type: ExtractMetadataOperationType = self._kiara.operation_registry.get_operation_type("extract_metadata") # type: ignore
operations = op_type.get_operations_for_data_type(value.value_schema.type)
for metadata_key, op in operations.items():
op_details: ExtractMetadataDetails = op.operation_details # type: ignore
input_field_name = op_details.input_field_name
result_field_name = op_details.result_field_name
self._kiara.destiny_registry.add_destiny(
destiny_alias=f"metadata.{metadata_key}",
values={input_field_name: value.value_id},
manifest=op,
result_field_name=result_field_name,
)
def resolve_all_metadata(self, value: Value):
if self._skip_internal_types:
lineage = self._kiara.type_registry.get_type_lineage(
value.value_schema.type
)
if "any" not in lineage:
return
assert not value.is_stored
aliases = self._kiara.destiny_registry.get_destiny_aliases_for_value(
value_id=value.value_id
)
for alias in aliases:
destiny = self._kiara.destiny_registry.get_destiny(
value_id=value.value_id, destiny_alias=alias
)
self._kiara.destiny_registry.resolve_destiny(destiny)
self._kiara.destiny_registry.attach_as_property(destiny)
attach_metadata(self, value)
¶Source code in kiara/registries/events/metadata.py
def attach_metadata(self, value: Value):
assert not value.is_stored
if self._skip_internal_types:
if value.value_schema.type == "any":
return
lineage = self._kiara.type_registry.get_type_lineage(
value.value_schema.type
)
if "any" not in lineage:
return
op_type: ExtractMetadataOperationType = self._kiara.operation_registry.get_operation_type("extract_metadata") # type: ignore
operations = op_type.get_operations_for_data_type(value.value_schema.type)
for metadata_key, op in operations.items():
op_details: ExtractMetadataDetails = op.operation_details # type: ignore
input_field_name = op_details.input_field_name
result_field_name = op_details.result_field_name
self._kiara.destiny_registry.add_destiny(
destiny_alias=f"metadata.{metadata_key}",
values={input_field_name: value.value_id},
manifest=op,
result_field_name=result_field_name,
)
handle_events(self, *events)
¶Source code in kiara/registries/events/metadata.py
def handle_events(self, *events: KiaraEvent) -> Any:
for event in events:
if event.get_event_type() == "value_created": # type: ignore
if event.value.is_set: # type: ignore
self.attach_metadata(event.value) # type: ignore
for event in events:
if event.get_event_type() == "value_registered": # type: ignore
self.resolve_all_metadata(event.value) # type: ignore
resolve_all_metadata(self, value)
¶Source code in kiara/registries/events/metadata.py
def resolve_all_metadata(self, value: Value):
if self._skip_internal_types:
lineage = self._kiara.type_registry.get_type_lineage(
value.value_schema.type
)
if "any" not in lineage:
return
assert not value.is_stored
aliases = self._kiara.destiny_registry.get_destiny_aliases_for_value(
value_id=value.value_id
)
for alias in aliases:
destiny = self._kiara.destiny_registry.get_destiny(
value_id=value.value_id, destiny_alias=alias
)
self._kiara.destiny_registry.resolve_destiny(destiny)
self._kiara.destiny_registry.attach_as_property(destiny)
supported_event_types(self)
¶Source code in kiara/registries/events/metadata.py
def supported_event_types(self) -> Iterable[str]:
return ["value_created", "value_registered"]
registry
¶
AllEvents (KiaraEvent)
pydantic-model
¶Source code in kiara/registries/events/registry.py
class AllEvents(KiaraEvent):
pass
EventRegistry
¶Source code in kiara/registries/events/registry.py
class EventRegistry(object):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._producers: Dict[uuid.UUID, EventProducer] = {}
self._listeners: Dict[uuid.UUID, EventListener] = {}
self._subscriptions: Dict[uuid.UUID, List[str]] = {}
def add_producer(self, producer: EventProducer) -> Callable:
producer_id = ID_REGISTRY.generate(
obj=producer, comment="adding event producer"
)
func = partial(self.handle_events, producer_id)
return func
def add_listener(self, listener, *subscriptions: str):
if not subscriptions:
_subscriptions = ["*"]
else:
_subscriptions = list(subscriptions)
listener_id = ID_REGISTRY.generate(
obj=listener, comment="adding event listener"
)
self._listeners[listener_id] = listener
self._subscriptions[listener_id] = _subscriptions
def _matches_subscription(
self, events: Iterable[KiaraEvent], subscriptions: Iterable[str]
) -> Iterable[KiaraEvent]:
result = []
for subscription in subscriptions:
for event in events:
match = fnmatch.filter([event.get_event_type()], subscription)
if match:
result.append(event)
return result
def handle_events(self, producer_id: uuid.UUID, *events: KiaraEvent):
event_targets: Dict[uuid.UUID, List[KiaraEvent]] = {}
for l_id, listener in self._listeners.items():
matches = self._matches_subscription(
events=events, subscriptions=self._subscriptions[l_id]
)
if matches:
event_targets.setdefault(l_id, []).extend(matches)
responses = {}
for l_id, l_events in event_targets.items():
listener = self._listeners[l_id]
response = listener.handle_events(*l_events)
responses[l_id] = response
for l_id, response in responses.items():
if response is None:
continue
a_listener: AsyncEventListener = self._listeners[l_id] # type: ignore
if not hasattr(a_listener, "wait_for_processing"):
raise Exception(
"Can't wait for processing of event for listener: listener does not provide 'wait_for_processing' method."
)
a_listener.wait_for_processing(response)
add_listener(self, listener, *subscriptions)
¶Source code in kiara/registries/events/registry.py
def add_listener(self, listener, *subscriptions: str):
if not subscriptions:
_subscriptions = ["*"]
else:
_subscriptions = list(subscriptions)
listener_id = ID_REGISTRY.generate(
obj=listener, comment="adding event listener"
)
self._listeners[listener_id] = listener
self._subscriptions[listener_id] = _subscriptions
add_producer(self, producer)
¶Source code in kiara/registries/events/registry.py
def add_producer(self, producer: EventProducer) -> Callable:
producer_id = ID_REGISTRY.generate(
obj=producer, comment="adding event producer"
)
func = partial(self.handle_events, producer_id)
return func
handle_events(self, producer_id, *events)
¶Source code in kiara/registries/events/registry.py
def handle_events(self, producer_id: uuid.UUID, *events: KiaraEvent):
event_targets: Dict[uuid.UUID, List[KiaraEvent]] = {}
for l_id, listener in self._listeners.items():
matches = self._matches_subscription(
events=events, subscriptions=self._subscriptions[l_id]
)
if matches:
event_targets.setdefault(l_id, []).extend(matches)
responses = {}
for l_id, l_events in event_targets.items():
listener = self._listeners[l_id]
response = listener.handle_events(*l_events)
responses[l_id] = response
for l_id, response in responses.items():
if response is None:
continue
a_listener: AsyncEventListener = self._listeners[l_id] # type: ignore
if not hasattr(a_listener, "wait_for_processing"):
raise Exception(
"Can't wait for processing of event for listener: listener does not provide 'wait_for_processing' method."
)
a_listener.wait_for_processing(response)
ids
special
¶
ID_REGISTRY
¶logger
¶
IdRegistry
¶Source code in kiara/registries/ids/__init__.py
class IdRegistry(object):
def __init__(self):
self._ids: Dict[uuid.UUID, Dict[Type, Dict[str, Any]]] = {}
self._objs: Dict[uuid.UUID, WeakValueDictionary[Type, Any]] = {}
def generate(
self,
id: Union[uuid.UUID, None] = None,
obj_type: Union[Type, None] = None,
obj: Union[Any, None] = None,
**metadata: Any
):
if id is None:
id = uuid.uuid4()
if is_debug() or is_develop():
# logger.debug("generate.id", id=id, metadata=metadata)
if obj_type is None:
if obj:
obj_type = obj.__class__
else:
obj_type = NO_TYPE_MARKER
self._ids.setdefault(id, {}).setdefault(obj_type, {}).update(metadata)
if obj:
self._objs.setdefault(id, WeakValueDictionary())[obj_type] = obj
return id
def update_metadata(
self,
id: uuid.UUID,
obj_type: Union[Type, None] = None,
obj: Union[Any, None] = None,
**metadata
):
if not is_debug() and not is_develop():
return
if obj_type is None:
if obj:
obj_type = obj.__class__
else:
obj_type = NO_TYPE_MARKER
self._ids.setdefault(id, {}).setdefault(obj_type, {}).update(metadata)
if obj:
self._objs.setdefault(id, WeakValueDictionary())[obj_type] = obj
generate(self, id=None, obj_type=None, obj=None, **metadata)
¶Source code in kiara/registries/ids/__init__.py
def generate(
self,
id: Union[uuid.UUID, None] = None,
obj_type: Union[Type, None] = None,
obj: Union[Any, None] = None,
**metadata: Any
):
if id is None:
id = uuid.uuid4()
if is_debug() or is_develop():
# logger.debug("generate.id", id=id, metadata=metadata)
if obj_type is None:
if obj:
obj_type = obj.__class__
else:
obj_type = NO_TYPE_MARKER
self._ids.setdefault(id, {}).setdefault(obj_type, {}).update(metadata)
if obj:
self._objs.setdefault(id, WeakValueDictionary())[obj_type] = obj
return id
update_metadata(self, id, obj_type=None, obj=None, **metadata)
¶Source code in kiara/registries/ids/__init__.py
def update_metadata(
self,
id: uuid.UUID,
obj_type: Union[Type, None] = None,
obj: Union[Any, None] = None,
**metadata
):
if not is_debug() and not is_develop():
return
if obj_type is None:
if obj:
obj_type = obj.__class__
else:
obj_type = NO_TYPE_MARKER
self._ids.setdefault(id, {}).setdefault(obj_type, {}).update(metadata)
if obj:
self._objs.setdefault(id, WeakValueDictionary())[obj_type] = obj
NO_TYPE_MARKER
¶Source code in kiara/registries/ids/__init__.py
class NO_TYPE_MARKER(object):
pass
jobs
special
¶
MANIFEST_SUB_PATH
¶logger
¶Classes¶
DataHashJobMatcher (JobMatcher)
¶Source code in kiara/registries/jobs/__init__.py
class DataHashJobMatcher(JobMatcher):
def find_existing_job(
self, inputs_manifest: InputsManifest
) -> Union[JobRecord, None]:
matches = []
for store_id, archive in self._kiara.job_registry.job_archives.items():
match = archive.retrieve_record_for_job_hash(
job_hash=inputs_manifest.job_hash
)
if match:
matches.append(match)
if len(matches) > 1:
raise Exception(
f"Multiple stores have a record for inputs manifest '{inputs_manifest}', this is not supported (yet)."
)
elif len(matches) == 1:
job_record = matches[0]
job_record._is_stored = True
return job_record
inputs_data_cid = inputs_manifest.calculate_inputs_data_cid(
data_registry=self._kiara.data_registry
)
if not inputs_data_cid:
return None
inputs_data_hash = str(inputs_data_cid)
matching_records = []
for store_id, archive in self._kiara.job_registry.job_archives.items():
_matches = archive.retrieve_all_job_hashes(
manifest_hash=inputs_manifest.manifest_hash
)
for _match in _matches:
_job_record = archive.retrieve_record_for_job_hash(_match)
assert _job_record is not None
if _job_record.inputs_data_hash == inputs_data_hash:
matching_records.append(_job_record)
if not matching_records:
return None
elif len(matches) > 1:
raise Exception(
f"Multiple stores have a record for inputs manifest '{inputs_manifest}', this is not supported (yet)."
)
else:
return matching_records[0]
find_existing_job(self, inputs_manifest)
¶Source code in kiara/registries/jobs/__init__.py
def find_existing_job(
self, inputs_manifest: InputsManifest
) -> Union[JobRecord, None]:
matches = []
for store_id, archive in self._kiara.job_registry.job_archives.items():
match = archive.retrieve_record_for_job_hash(
job_hash=inputs_manifest.job_hash
)
if match:
matches.append(match)
if len(matches) > 1:
raise Exception(
f"Multiple stores have a record for inputs manifest '{inputs_manifest}', this is not supported (yet)."
)
elif len(matches) == 1:
job_record = matches[0]
job_record._is_stored = True
return job_record
inputs_data_cid = inputs_manifest.calculate_inputs_data_cid(
data_registry=self._kiara.data_registry
)
if not inputs_data_cid:
return None
inputs_data_hash = str(inputs_data_cid)
matching_records = []
for store_id, archive in self._kiara.job_registry.job_archives.items():
_matches = archive.retrieve_all_job_hashes(
manifest_hash=inputs_manifest.manifest_hash
)
for _match in _matches:
_job_record = archive.retrieve_record_for_job_hash(_match)
assert _job_record is not None
if _job_record.inputs_data_hash == inputs_data_hash:
matching_records.append(_job_record)
if not matching_records:
return None
elif len(matches) > 1:
raise Exception(
f"Multiple stores have a record for inputs manifest '{inputs_manifest}', this is not supported (yet)."
)
else:
return matching_records[0]
JobArchive (BaseArchive)
¶Source code in kiara/registries/jobs/__init__.py
class JobArchive(BaseArchive):
# @abc.abstractmethod
# def find_matching_job_record(
# self, inputs_manifest: InputsManifest
# ) -> Optional[JobRecord]:
# pass
@abc.abstractmethod
def retrieve_all_job_hashes(
self,
manifest_hash: Union[str, None] = None,
inputs_hash: Union[str, None] = None,
) -> Iterable[str]:
"""Retrieve a list of all job record hashes (cids) that match the given filter arguments.
A job record hash includes information about the module type used in the job, the module configuration, as well as input field names and value ids for the values used in those inputs.
If the job archive retrieves its jobs in a dynamic way, this will return 'None'.
"""
@abc.abstractmethod
def _retrieve_record_for_job_hash(self, job_hash: str) -> Union[JobRecord, None]:
pass
def retrieve_record_for_job_hash(self, job_hash: str) -> Union[JobRecord, None]:
job_record = self._retrieve_record_for_job_hash(job_hash=job_hash)
return job_record
retrieve_all_job_hashes(self, manifest_hash=None, inputs_hash=None)
¶Retrieve a list of all job record hashes (cids) that match the given filter arguments.
A job record hash includes information about the module type used in the job, the module configuration, as well as input field names and value ids for the values used in those inputs.
If the job archive retrieves its jobs in a dynamic way, this will return 'None'.
Source code in kiara/registries/jobs/__init__.py
@abc.abstractmethod
def retrieve_all_job_hashes(
self,
manifest_hash: Union[str, None] = None,
inputs_hash: Union[str, None] = None,
) -> Iterable[str]:
"""Retrieve a list of all job record hashes (cids) that match the given filter arguments.
A job record hash includes information about the module type used in the job, the module configuration, as well as input field names and value ids for the values used in those inputs.
If the job archive retrieves its jobs in a dynamic way, this will return 'None'.
"""
retrieve_record_for_job_hash(self, job_hash)
¶Source code in kiara/registries/jobs/__init__.py
def retrieve_record_for_job_hash(self, job_hash: str) -> Union[JobRecord, None]:
job_record = self._retrieve_record_for_job_hash(job_hash=job_hash)
return job_record
JobMatcher (ABC)
¶Source code in kiara/registries/jobs/__init__.py
class JobMatcher(abc.ABC):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
@abc.abstractmethod
def find_existing_job(
self, inputs_manifest: InputsManifest
) -> Union[JobRecord, None]:
pass
find_existing_job(self, inputs_manifest)
¶Source code in kiara/registries/jobs/__init__.py
@abc.abstractmethod
def find_existing_job(
self, inputs_manifest: InputsManifest
) -> Union[JobRecord, None]:
pass
JobRegistry
¶Source code in kiara/registries/jobs/__init__.py
class JobRegistry(object):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._job_matcher_cache: Dict[JobCacheStrategy, JobMatcher] = {}
self._active_jobs: bidict[str, uuid.UUID] = bidict()
self._failed_jobs: Dict[str, uuid.UUID] = {}
self._finished_jobs: Dict[str, uuid.UUID] = {}
self._archived_records: Dict[uuid.UUID, JobRecord] = {}
self._processor: ModuleProcessor = SynchronousProcessor(kiara=self._kiara)
self._processor.register_job_status_listener(self)
self._job_archives: Dict[str, JobArchive] = {}
self._default_job_store: Union[str, None] = None
self._event_callback = self._kiara.event_registry.add_producer(self)
# default_archive = FileSystemJobStore.create_from_kiara_context(self._kiara)
# self.register_job_archive(default_archive, store_alias=DEFAULT_STORE_MARKER)
# default_file_store = self._kiara.data_registry.get_archive(DEFAULT_STORE_MARKER)
# self.register_job_archive(default_file_store, store_alias="default_data_store") # type: ignore
@property
def job_matcher(self) -> JobMatcher:
strategy = self._kiara.runtime_config.job_cache
if is_develop():
dev_config = get_dev_config()
if not dev_config.job_cache:
logger.debug(
"disable.job_cache",
reason="dev mode enabled and 'disable_job_cache' is set.",
)
strategy = JobCacheStrategy.no_cache
job_matcher = self._job_matcher_cache.get(strategy, None)
if job_matcher is None:
if strategy == JobCacheStrategy.no_cache:
job_matcher = NoneJobMatcher(kiara=self._kiara)
elif strategy == JobCacheStrategy.value_id:
job_matcher = ValueIdJobMatcher(kiara=self._kiara)
elif strategy == JobCacheStrategy.data_hash:
job_matcher = DataHashJobMatcher(kiara=self._kiara)
else:
raise Exception(f"Job cache strategy not implemented: {strategy}")
self._job_matcher_cache[strategy] = job_matcher
return job_matcher
def suppoerted_event_types(self) -> Iterable[Type[KiaraEvent]]:
return [JobArchiveAddedEvent, JobRecordPreStoreEvent, JobRecordStoredEvent]
def register_job_archive(self, archive: JobArchive, alias: Union[str, None] = None):
if alias is None:
alias = str(archive.archive_id)
if alias in self._job_archives.keys():
raise Exception(
f"Can't register job store, store id already registered: {alias}."
)
self._job_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, JobStore):
is_store = True
if self._default_job_store is None:
self._default_job_store = alias
event = JobArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
job_archive_id=archive.archive_id,
job_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
@property
def default_job_store(self) -> str:
if self._default_job_store is None:
raise Exception("No default job store set (yet).")
return self._default_job_store # type: ignore
def get_archive(self, store_id: Union[str, None] = None) -> JobArchive:
if store_id is None:
store_id = self.default_job_store
if store_id is None:
raise Exception("Can't retrieve deafult job archive, none set (yet).")
return self._job_archives[store_id]
@property
def job_archives(self) -> Mapping[str, JobArchive]:
return self._job_archives
def job_status_changed(
self,
job_id: uuid.UUID,
old_status: Union[JobStatus, None],
new_status: JobStatus,
):
# print(f"JOB STATUS CHANGED: {job_id} - {old_status} - {new_status.value}")
if job_id in self._active_jobs.values() and new_status is JobStatus.FAILED:
job_hash = self._active_jobs.inverse.pop(job_id)
self._failed_jobs[job_hash] = job_id
elif job_id in self._active_jobs.values() and new_status is JobStatus.SUCCESS:
job_hash = self._active_jobs.inverse.pop(job_id)
job_record = self._processor.get_job_record(job_id)
self._finished_jobs[job_hash] = job_id
self._archived_records[job_id] = job_record
def store_job_record(self, job_id: uuid.UUID):
if job_id not in self._archived_records.keys():
raise Exception(
f"Can't store job with id '{job_id}': no job record with that id exists."
)
job_record = self._archived_records[job_id]
if job_record._is_stored:
logger.debug(
"ignore.store.job_record", reason="already stored", job_id=str(job_id)
)
return
store: JobStore = self.get_archive() # type: ignore
if not isinstance(store, JobStore):
raise Exception("Can't store job record to archive: not writable.")
# if job_record.job_id in self._finished_jobs.values():
# logger.debug(
# "ignore.store.job_record",
# reason="already stored in store",
# job_id=str(job_id),
# )
# return
logger.debug(
"store.job_record",
job_hash=job_record.job_hash,
module_type=job_record.module_type,
)
pre_store_event = JobRecordPreStoreEvent.construct(
kiara_id=self._kiara.id, job_record=job_record
)
self._event_callback(pre_store_event)
store.store_job_record(job_record)
stored_event = JobRecordStoredEvent.construct(
kiara_id=self._kiara.id, job_record=job_record
)
self._event_callback(stored_event)
def get_job_record_in_session(self, job_id: uuid.UUID) -> JobRecord:
return self._processor.get_job_record(job_id)
def get_job_record(self, job_id: uuid.UUID) -> Union[JobRecord, None]:
if job_id in self._archived_records.keys():
return self._archived_records[job_id]
try:
job_record = self._processor.get_job_record(job_id=job_id)
return job_record
except Exception:
pass
job = self._processor.get_job(job_id=job_id)
if job is not None:
if job.status == JobStatus.FAILED:
return None
raise NotImplementedError()
def retrieve_all_job_records(self) -> Mapping[str, JobRecord]:
all_records: Dict[str, JobRecord] = {}
for archive in self.job_archives.values():
all_record_ids = archive.retrieve_all_job_hashes()
if all_record_ids is None:
return {}
for r in all_record_ids:
assert r not in all_records.keys()
job_record = archive.retrieve_record_for_job_hash(r)
assert job_record is not None
all_records[r] = job_record
return all_records
def find_matching_job_record(
self, inputs_manifest: InputsManifest
) -> Union[uuid.UUID, None]:
"""Check if a job with same inputs manifest already ran some time before.
Arguments:
inputs_manifest: the manifest incl. inputs
Returns:
'None' if no such job exists, a (uuid) job-id if the job is currently running or has run in the past
"""
log = logger.bind(module_type=inputs_manifest.module_type)
if inputs_manifest.job_hash in self._active_jobs.keys():
log.debug("job.use_running")
return self._active_jobs[inputs_manifest.job_hash]
if inputs_manifest.job_hash in self._finished_jobs.keys():
job_id = self._finished_jobs[inputs_manifest.job_hash]
return job_id
module = self._kiara.module_registry.create_module(manifest=inputs_manifest)
if not module.characteristics.is_idempotent:
log.debug(
"skip.job_matching",
reason="module is not idempotent",
module_type=inputs_manifest.module_type,
)
return None
job_record = self.job_matcher.find_existing_job(inputs_manifest=inputs_manifest)
if job_record is None:
return None
self._finished_jobs[inputs_manifest.job_hash] = job_record.job_id
self._archived_records[job_record.job_id] = job_record
log.debug(
"job.found_cached_record",
job_id=str(job_record.job_id),
job_hash=inputs_manifest.job_hash,
module_type=inputs_manifest.module_type,
)
return job_record.job_id
def prepare_job_config(
self, manifest: Manifest, inputs: Mapping[str, Any]
) -> JobConfig:
module = self._kiara.create_module(manifest=manifest)
job_config = JobConfig.create_from_module(
data_registry=self._kiara.data_registry, module=module, inputs=inputs
)
return job_config
def execute(
self,
manifest: Manifest,
inputs: Mapping[str, Any],
wait: bool = False,
job_metadata: Union[None, Any] = None,
) -> uuid.UUID:
job_config = self.prepare_job_config(manifest=manifest, inputs=inputs)
return self.execute_job(job_config, wait=wait, job_metadata=job_metadata)
def execute_job(
self,
job_config: JobConfig,
wait: bool = False,
job_metadata: Union[None, Any] = None,
) -> uuid.UUID:
log = logger.bind(
module_type=job_config.module_type,
module_config=job_config.module_config,
inputs={k: str(v) for k, v in job_config.inputs.items()},
job_hash=job_config.job_hash,
)
stored_job = self.find_matching_job_record(inputs_manifest=job_config)
if stored_job is not None:
log.debug(
"job.use_cached",
job_id=str(stored_job),
module_type=job_config.module_type,
)
return stored_job
if job_metadata is None:
job_metadata = {}
is_pipeline_step = job_metadata.get("is_pipeline_step", False)
dbg_data = {
"module_type": job_config.module_type,
"is_pipeline_step": is_pipeline_step,
}
if is_pipeline_step:
dbg_data["step_id"] = job_metadata["step_id"]
log.debug("job.execute", **dbg_data)
job_id = self._processor.create_job(
job_config=job_config, job_metadata=job_metadata
)
self._active_jobs[job_config.job_hash] = job_id
try:
self._processor.queue_job(job_id=job_id)
except Exception as e:
log.error("error.queue_job", job_id=job_id)
raise e
if wait:
self._processor.wait_for(job_id)
return job_id
def get_active_job(self, job_id: uuid.UUID) -> ActiveJob:
if job_id in self._active_jobs.keys() or job_id in self._failed_jobs.keys():
return self._processor.get_job(job_id)
else:
if job_id in self._archived_records.keys():
raise Exception(
f"Can't retrieve active job with id '{job_id}': job is archived."
)
elif job_id in self._processor._failed_jobs.keys():
job = self._processor.get_job(job_id)
msg = job.error
if not msg and job._exception:
msg = str(job._exception)
if not msg:
msg = repr(job._exception)
raise FailedJobException(job=job, msg=msg)
else:
raise Exception(f"Can't retrieve job with id '{job_id}': no such job.")
def get_job(self, job_id: uuid.UUID) -> ActiveJob:
return self._processor.get_job(job_id=job_id)
def get_job_status(self, job_id: uuid.UUID) -> JobStatus:
if job_id in self._archived_records.keys():
return JobStatus.SUCCESS
elif job_id in self._failed_jobs.values():
return JobStatus.FAILED
return self._processor.get_job_status(job_id=job_id)
def wait_for(self, *job_id: uuid.UUID):
not_finished = (j for j in job_id if j not in self._archived_records.keys())
if not_finished:
self._processor.wait_for(*not_finished)
def retrieve_result(self, job_id: uuid.UUID) -> ValueMap:
if job_id not in self._archived_records.keys():
self._processor.wait_for(job_id)
if job_id in self._archived_records.keys():
job_record = self._archived_records[job_id]
results = self._kiara.data_registry.load_values(job_record.outputs)
return results
elif job_id in self._failed_jobs.values():
j = self._processor.get_job(job_id=job_id)
raise Exception(f"Job failed: {j.error}")
else:
raise Exception(f"Could not find job with id: {job_id}")
def execute_and_retrieve(
self, manifest: Manifest, inputs: Mapping[str, Any]
) -> ValueMap:
job_id = self.execute(manifest=manifest, inputs=inputs, wait=True)
results = self.retrieve_result(job_id=job_id)
return results
default_job_store: str
property
readonly
¶job_archives: Mapping[str, kiara.registries.jobs.JobArchive]
property
readonly
¶job_matcher: JobMatcher
property
readonly
¶execute(self, manifest, inputs, wait=False, job_metadata=None)
¶Source code in kiara/registries/jobs/__init__.py
def execute(
self,
manifest: Manifest,
inputs: Mapping[str, Any],
wait: bool = False,
job_metadata: Union[None, Any] = None,
) -> uuid.UUID:
job_config = self.prepare_job_config(manifest=manifest, inputs=inputs)
return self.execute_job(job_config, wait=wait, job_metadata=job_metadata)
execute_and_retrieve(self, manifest, inputs)
¶Source code in kiara/registries/jobs/__init__.py
def execute_and_retrieve(
self, manifest: Manifest, inputs: Mapping[str, Any]
) -> ValueMap:
job_id = self.execute(manifest=manifest, inputs=inputs, wait=True)
results = self.retrieve_result(job_id=job_id)
return results
execute_job(self, job_config, wait=False, job_metadata=None)
¶Source code in kiara/registries/jobs/__init__.py
def execute_job(
self,
job_config: JobConfig,
wait: bool = False,
job_metadata: Union[None, Any] = None,
) -> uuid.UUID:
log = logger.bind(
module_type=job_config.module_type,
module_config=job_config.module_config,
inputs={k: str(v) for k, v in job_config.inputs.items()},
job_hash=job_config.job_hash,
)
stored_job = self.find_matching_job_record(inputs_manifest=job_config)
if stored_job is not None:
log.debug(
"job.use_cached",
job_id=str(stored_job),
module_type=job_config.module_type,
)
return stored_job
if job_metadata is None:
job_metadata = {}
is_pipeline_step = job_metadata.get("is_pipeline_step", False)
dbg_data = {
"module_type": job_config.module_type,
"is_pipeline_step": is_pipeline_step,
}
if is_pipeline_step:
dbg_data["step_id"] = job_metadata["step_id"]
log.debug("job.execute", **dbg_data)
job_id = self._processor.create_job(
job_config=job_config, job_metadata=job_metadata
)
self._active_jobs[job_config.job_hash] = job_id
try:
self._processor.queue_job(job_id=job_id)
except Exception as e:
log.error("error.queue_job", job_id=job_id)
raise e
if wait:
self._processor.wait_for(job_id)
return job_id
find_matching_job_record(self, inputs_manifest)
¶Check if a job with same inputs manifest already ran some time before.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
inputs_manifest |
InputsManifest |
the manifest incl. inputs |
required |
Returns:
| Type | Description |
|---|---|
Optional[uuid.UUID] |
'None' if no such job exists, a (uuid) job-id if the job is currently running or has run in the past |
Source code in kiara/registries/jobs/__init__.py
def find_matching_job_record(
self, inputs_manifest: InputsManifest
) -> Union[uuid.UUID, None]:
"""Check if a job with same inputs manifest already ran some time before.
Arguments:
inputs_manifest: the manifest incl. inputs
Returns:
'None' if no such job exists, a (uuid) job-id if the job is currently running or has run in the past
"""
log = logger.bind(module_type=inputs_manifest.module_type)
if inputs_manifest.job_hash in self._active_jobs.keys():
log.debug("job.use_running")
return self._active_jobs[inputs_manifest.job_hash]
if inputs_manifest.job_hash in self._finished_jobs.keys():
job_id = self._finished_jobs[inputs_manifest.job_hash]
return job_id
module = self._kiara.module_registry.create_module(manifest=inputs_manifest)
if not module.characteristics.is_idempotent:
log.debug(
"skip.job_matching",
reason="module is not idempotent",
module_type=inputs_manifest.module_type,
)
return None
job_record = self.job_matcher.find_existing_job(inputs_manifest=inputs_manifest)
if job_record is None:
return None
self._finished_jobs[inputs_manifest.job_hash] = job_record.job_id
self._archived_records[job_record.job_id] = job_record
log.debug(
"job.found_cached_record",
job_id=str(job_record.job_id),
job_hash=inputs_manifest.job_hash,
module_type=inputs_manifest.module_type,
)
return job_record.job_id
get_active_job(self, job_id)
¶Source code in kiara/registries/jobs/__init__.py
def get_active_job(self, job_id: uuid.UUID) -> ActiveJob:
if job_id in self._active_jobs.keys() or job_id in self._failed_jobs.keys():
return self._processor.get_job(job_id)
else:
if job_id in self._archived_records.keys():
raise Exception(
f"Can't retrieve active job with id '{job_id}': job is archived."
)
elif job_id in self._processor._failed_jobs.keys():
job = self._processor.get_job(job_id)
msg = job.error
if not msg and job._exception:
msg = str(job._exception)
if not msg:
msg = repr(job._exception)
raise FailedJobException(job=job, msg=msg)
else:
raise Exception(f"Can't retrieve job with id '{job_id}': no such job.")
get_archive(self, store_id=None)
¶Source code in kiara/registries/jobs/__init__.py
def get_archive(self, store_id: Union[str, None] = None) -> JobArchive:
if store_id is None:
store_id = self.default_job_store
if store_id is None:
raise Exception("Can't retrieve deafult job archive, none set (yet).")
return self._job_archives[store_id]
get_job(self, job_id)
¶Source code in kiara/registries/jobs/__init__.py
def get_job(self, job_id: uuid.UUID) -> ActiveJob:
return self._processor.get_job(job_id=job_id)
get_job_record(self, job_id)
¶Source code in kiara/registries/jobs/__init__.py
def get_job_record(self, job_id: uuid.UUID) -> Union[JobRecord, None]:
if job_id in self._archived_records.keys():
return self._archived_records[job_id]
try:
job_record = self._processor.get_job_record(job_id=job_id)
return job_record
except Exception:
pass
job = self._processor.get_job(job_id=job_id)
if job is not None:
if job.status == JobStatus.FAILED:
return None
raise NotImplementedError()
get_job_record_in_session(self, job_id)
¶Source code in kiara/registries/jobs/__init__.py
def get_job_record_in_session(self, job_id: uuid.UUID) -> JobRecord:
return self._processor.get_job_record(job_id)
get_job_status(self, job_id)
¶Source code in kiara/registries/jobs/__init__.py
def get_job_status(self, job_id: uuid.UUID) -> JobStatus:
if job_id in self._archived_records.keys():
return JobStatus.SUCCESS
elif job_id in self._failed_jobs.values():
return JobStatus.FAILED
return self._processor.get_job_status(job_id=job_id)
job_status_changed(self, job_id, old_status, new_status)
¶Source code in kiara/registries/jobs/__init__.py
def job_status_changed(
self,
job_id: uuid.UUID,
old_status: Union[JobStatus, None],
new_status: JobStatus,
):
# print(f"JOB STATUS CHANGED: {job_id} - {old_status} - {new_status.value}")
if job_id in self._active_jobs.values() and new_status is JobStatus.FAILED:
job_hash = self._active_jobs.inverse.pop(job_id)
self._failed_jobs[job_hash] = job_id
elif job_id in self._active_jobs.values() and new_status is JobStatus.SUCCESS:
job_hash = self._active_jobs.inverse.pop(job_id)
job_record = self._processor.get_job_record(job_id)
self._finished_jobs[job_hash] = job_id
self._archived_records[job_id] = job_record
prepare_job_config(self, manifest, inputs)
¶Source code in kiara/registries/jobs/__init__.py
def prepare_job_config(
self, manifest: Manifest, inputs: Mapping[str, Any]
) -> JobConfig:
module = self._kiara.create_module(manifest=manifest)
job_config = JobConfig.create_from_module(
data_registry=self._kiara.data_registry, module=module, inputs=inputs
)
return job_config
register_job_archive(self, archive, alias=None)
¶Source code in kiara/registries/jobs/__init__.py
def register_job_archive(self, archive: JobArchive, alias: Union[str, None] = None):
if alias is None:
alias = str(archive.archive_id)
if alias in self._job_archives.keys():
raise Exception(
f"Can't register job store, store id already registered: {alias}."
)
self._job_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, JobStore):
is_store = True
if self._default_job_store is None:
self._default_job_store = alias
event = JobArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
job_archive_id=archive.archive_id,
job_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
retrieve_all_job_records(self)
¶Source code in kiara/registries/jobs/__init__.py
def retrieve_all_job_records(self) -> Mapping[str, JobRecord]:
all_records: Dict[str, JobRecord] = {}
for archive in self.job_archives.values():
all_record_ids = archive.retrieve_all_job_hashes()
if all_record_ids is None:
return {}
for r in all_record_ids:
assert r not in all_records.keys()
job_record = archive.retrieve_record_for_job_hash(r)
assert job_record is not None
all_records[r] = job_record
return all_records
retrieve_result(self, job_id)
¶Source code in kiara/registries/jobs/__init__.py
def retrieve_result(self, job_id: uuid.UUID) -> ValueMap:
if job_id not in self._archived_records.keys():
self._processor.wait_for(job_id)
if job_id in self._archived_records.keys():
job_record = self._archived_records[job_id]
results = self._kiara.data_registry.load_values(job_record.outputs)
return results
elif job_id in self._failed_jobs.values():
j = self._processor.get_job(job_id=job_id)
raise Exception(f"Job failed: {j.error}")
else:
raise Exception(f"Could not find job with id: {job_id}")
store_job_record(self, job_id)
¶Source code in kiara/registries/jobs/__init__.py
def store_job_record(self, job_id: uuid.UUID):
if job_id not in self._archived_records.keys():
raise Exception(
f"Can't store job with id '{job_id}': no job record with that id exists."
)
job_record = self._archived_records[job_id]
if job_record._is_stored:
logger.debug(
"ignore.store.job_record", reason="already stored", job_id=str(job_id)
)
return
store: JobStore = self.get_archive() # type: ignore
if not isinstance(store, JobStore):
raise Exception("Can't store job record to archive: not writable.")
# if job_record.job_id in self._finished_jobs.values():
# logger.debug(
# "ignore.store.job_record",
# reason="already stored in store",
# job_id=str(job_id),
# )
# return
logger.debug(
"store.job_record",
job_hash=job_record.job_hash,
module_type=job_record.module_type,
)
pre_store_event = JobRecordPreStoreEvent.construct(
kiara_id=self._kiara.id, job_record=job_record
)
self._event_callback(pre_store_event)
store.store_job_record(job_record)
stored_event = JobRecordStoredEvent.construct(
kiara_id=self._kiara.id, job_record=job_record
)
self._event_callback(stored_event)
suppoerted_event_types(self)
¶Source code in kiara/registries/jobs/__init__.py
def suppoerted_event_types(self) -> Iterable[Type[KiaraEvent]]:
return [JobArchiveAddedEvent, JobRecordPreStoreEvent, JobRecordStoredEvent]
wait_for(self, *job_id)
¶Source code in kiara/registries/jobs/__init__.py
def wait_for(self, *job_id: uuid.UUID):
not_finished = (j for j in job_id if j not in self._archived_records.keys())
if not_finished:
self._processor.wait_for(*not_finished)
JobStore (JobArchive)
¶Source code in kiara/registries/jobs/__init__.py
class JobStore(JobArchive):
@abc.abstractmethod
def store_job_record(self, job_record: JobRecord):
pass
store_job_record(self, job_record)
¶Source code in kiara/registries/jobs/__init__.py
@abc.abstractmethod
def store_job_record(self, job_record: JobRecord):
pass
NoneJobMatcher (JobMatcher)
¶Source code in kiara/registries/jobs/__init__.py
class NoneJobMatcher(JobMatcher):
def find_existing_job(
self, inputs_manifest: InputsManifest
) -> Union[JobRecord, None]:
return None
find_existing_job(self, inputs_manifest)
¶Source code in kiara/registries/jobs/__init__.py
def find_existing_job(
self, inputs_manifest: InputsManifest
) -> Union[JobRecord, None]:
return None
ValueIdJobMatcher (JobMatcher)
¶Source code in kiara/registries/jobs/__init__.py
class ValueIdJobMatcher(JobMatcher):
def find_existing_job(
self, inputs_manifest: InputsManifest
) -> Union[JobRecord, None]:
matches = []
for store_id, archive in self._kiara.job_registry.job_archives.items():
match = archive.retrieve_record_for_job_hash(
job_hash=inputs_manifest.job_hash
)
if match:
matches.append(match)
if len(matches) == 0:
return None
elif len(matches) > 1:
raise Exception(
f"Multiple stores have a record for inputs manifest '{inputs_manifest}', this is not supported (yet)."
)
job_record = matches[0]
job_record._is_stored = True
return job_record
find_existing_job(self, inputs_manifest)
¶Source code in kiara/registries/jobs/__init__.py
def find_existing_job(
self, inputs_manifest: InputsManifest
) -> Union[JobRecord, None]:
matches = []
for store_id, archive in self._kiara.job_registry.job_archives.items():
match = archive.retrieve_record_for_job_hash(
job_hash=inputs_manifest.job_hash
)
if match:
matches.append(match)
if len(matches) == 0:
return None
elif len(matches) > 1:
raise Exception(
f"Multiple stores have a record for inputs manifest '{inputs_manifest}', this is not supported (yet)."
)
job_record = matches[0]
job_record._is_stored = True
return job_record
Modules¶
job_store
special
¶filesystem_store
¶log
¶
FileSystemJobArchive (JobArchive)
¶Source code in kiara/registries/jobs/job_store/filesystem_store.py
class FileSystemJobArchive(JobArchive):
_archive_type_name = "filesystem_job_archive"
_config_cls = FileSystemArchiveConfig
@classmethod
def is_writeable(cls) -> bool:
return False
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["job_record"]
def __init__(self, archive_id: uuid.UUID, config: FileSystemArchiveConfig):
super().__init__(archive_id=archive_id, config=config)
self._base_path: Union[Path, None] = None
def get_archive_details(self) -> ArchiveDetails:
size = sum(
f.stat().st_size for f in self.job_store_path.glob("**/*") if f.is_file()
)
return ArchiveDetails(size=size)
@property
def job_store_path(self) -> Path:
if self._base_path is not None:
return self._base_path
self._base_path = Path(self.config.archive_path).absolute() # type: ignore
self._base_path.mkdir(parents=True, exist_ok=True)
return self._base_path
def _delete_archive(self):
shutil.rmtree(self.job_store_path)
def retrieve_all_job_hashes(
self,
manifest_hash: Union[str, None] = None,
inputs_hash: Union[str, None] = None,
) -> Iterable[str]:
base_path = self.job_store_path / MANIFEST_SUB_PATH
if not manifest_hash:
if not inputs_hash:
records = base_path.glob("*/*/*.job_record")
else:
records = base_path.glob(f"*/{inputs_hash}/*.job_record")
else:
if not inputs_hash:
records = base_path.glob(f"{manifest_hash}/*/*.job_record")
else:
records = base_path.glob(f"{manifest_hash}/{inputs_hash}/*.job_record")
result = []
for record in records:
result.append(record.name[0:-11])
return result
def _retrieve_record_for_job_hash(self, job_hash: str) -> Union[JobRecord, None]:
base_path = self.job_store_path / MANIFEST_SUB_PATH
records = list(base_path.glob(f"*/*/{job_hash}.job_record"))
if not records:
return None
assert len(records) == 1
details_file = records[0]
details_content = details_file.read_text()
details: Dict[str, Any] = orjson.loads(details_content)
job_record = JobRecord(**details)
job_record._is_stored = True
return job_record
# def find_matching_job_record(
# self, inputs_manifest: InputsManifest
# ) -> Optional[JobRecord]:
#
# manifest_hash = inputs_manifest.manifest_cid
# jobs_hash = inputs_manifest.job_hash
#
# base_path = self.job_store_path / MANIFEST_SUB_PATH
# manifest_folder = base_path / str(manifest_hash)
#
# if not manifest_folder.exists():
# return None
#
# manifest_file = manifest_folder / "manifest.json"
#
# if not manifest_file.exists():
# raise Exception(
# f"No 'manifests.json' file for manifest with hash: {manifest_hash}"
# )
#
# details_folder = manifest_folder / str(jobs_hash)
# if not details_folder.exists():
# return None
#
# details_file_name = details_folder / "details.json"
# if not details_file_name.exists():
# raise Exception(
# f"No 'inputs.json' file for manifest/inputs hash-combo: {manifest_hash} / {jobs_hash}"
# )
#
# details_content = details_file_name.read_text()
# details: Dict[str, Any] = orjson.loads(details_content)
#
# job_record = JobRecord(**details)
# job_record._is_stored = True
# return job_record
job_store_path: Path
property
readonly
¶
_config_cls (ArchiveConfig)
private
pydantic-model
¶Source code in kiara/registries/jobs/job_store/filesystem_store.py
class FileSystemArchiveConfig(ArchiveConfig):
archive_path: str = Field(
description="The path where the data for this archive is stored."
)
get_archive_details(self)
¶Source code in kiara/registries/jobs/job_store/filesystem_store.py
def get_archive_details(self) -> ArchiveDetails:
size = sum(
f.stat().st_size for f in self.job_store_path.glob("**/*") if f.is_file()
)
return ArchiveDetails(size=size)
is_writeable()
classmethod
¶Source code in kiara/registries/jobs/job_store/filesystem_store.py
@classmethod
def is_writeable(cls) -> bool:
return False
retrieve_all_job_hashes(self, manifest_hash=None, inputs_hash=None)
¶Retrieve a list of all job record hashes (cids) that match the given filter arguments.
A job record hash includes information about the module type used in the job, the module configuration, as well as input field names and value ids for the values used in those inputs.
If the job archive retrieves its jobs in a dynamic way, this will return 'None'.
Source code in kiara/registries/jobs/job_store/filesystem_store.py
def retrieve_all_job_hashes(
self,
manifest_hash: Union[str, None] = None,
inputs_hash: Union[str, None] = None,
) -> Iterable[str]:
base_path = self.job_store_path / MANIFEST_SUB_PATH
if not manifest_hash:
if not inputs_hash:
records = base_path.glob("*/*/*.job_record")
else:
records = base_path.glob(f"*/{inputs_hash}/*.job_record")
else:
if not inputs_hash:
records = base_path.glob(f"{manifest_hash}/*/*.job_record")
else:
records = base_path.glob(f"{manifest_hash}/{inputs_hash}/*.job_record")
result = []
for record in records:
result.append(record.name[0:-11])
return result
supported_item_types()
classmethod
¶Source code in kiara/registries/jobs/job_store/filesystem_store.py
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["job_record"]
FileSystemJobStore (FileSystemJobArchive, JobStore)
¶Source code in kiara/registries/jobs/job_store/filesystem_store.py
class FileSystemJobStore(FileSystemJobArchive, JobStore):
_archive_type_name = "filesystem_job_store"
@classmethod
def is_writeable(cls) -> bool:
return False
def store_job_record(self, job_record: JobRecord):
manifest_cid = job_record.manifest_cid
inputs_hash = job_record.inputs_hash
base_path = self.job_store_path / MANIFEST_SUB_PATH
manifest_folder = base_path / str(manifest_cid)
manifest_folder.mkdir(parents=True, exist_ok=True)
manifest_info_file = manifest_folder / "manifest.json"
if not manifest_info_file.exists():
manifest_info_file.write_text(job_record.manifest_data_as_json())
job_folder = manifest_folder / inputs_hash
job_folder.mkdir(parents=True, exist_ok=True)
job_details_file = job_folder / f"{job_record.job_hash}.job_record"
exists = False
if job_details_file.exists():
exists = True
# TODO: check details match? or overwrite
file_m_time = datetime.datetime.fromtimestamp(
job_details_file.stat().st_mtime
).timestamp()
archive = job_folder / ".archive"
archive.mkdir(parents=True, exist_ok=True)
backup = archive / f"{job_details_file.name}.{file_m_time}"
log.debug(
"overwrite.store_job_record",
reason="job record already exists",
job_hash=job_record.job_hash,
new_path=backup.as_posix(),
)
shutil.move(job_details_file.as_posix(), backup)
job_details_file.write_text(job_record.json())
for output_name, output_v_id in job_record.outputs.items():
outputs_file_name = (
job_folder / f"output__{output_name}__value_id__{output_v_id}.json"
)
if outputs_file_name.exists() and not exists:
# if value.pedigree_output_name == "__void__":
# return
# else:
raise Exception(f"Can't write value '{output_v_id}': already exists.")
else:
outputs_file_name.touch()
is_writeable()
classmethod
¶Source code in kiara/registries/jobs/job_store/filesystem_store.py
@classmethod
def is_writeable(cls) -> bool:
return False
store_job_record(self, job_record)
¶Source code in kiara/registries/jobs/job_store/filesystem_store.py
def store_job_record(self, job_record: JobRecord):
manifest_cid = job_record.manifest_cid
inputs_hash = job_record.inputs_hash
base_path = self.job_store_path / MANIFEST_SUB_PATH
manifest_folder = base_path / str(manifest_cid)
manifest_folder.mkdir(parents=True, exist_ok=True)
manifest_info_file = manifest_folder / "manifest.json"
if not manifest_info_file.exists():
manifest_info_file.write_text(job_record.manifest_data_as_json())
job_folder = manifest_folder / inputs_hash
job_folder.mkdir(parents=True, exist_ok=True)
job_details_file = job_folder / f"{job_record.job_hash}.job_record"
exists = False
if job_details_file.exists():
exists = True
# TODO: check details match? or overwrite
file_m_time = datetime.datetime.fromtimestamp(
job_details_file.stat().st_mtime
).timestamp()
archive = job_folder / ".archive"
archive.mkdir(parents=True, exist_ok=True)
backup = archive / f"{job_details_file.name}.{file_m_time}"
log.debug(
"overwrite.store_job_record",
reason="job record already exists",
job_hash=job_record.job_hash,
new_path=backup.as_posix(),
)
shutil.move(job_details_file.as_posix(), backup)
job_details_file.write_text(job_record.json())
for output_name, output_v_id in job_record.outputs.items():
outputs_file_name = (
job_folder / f"output__{output_name}__value_id__{output_v_id}.json"
)
if outputs_file_name.exists() and not exists:
# if value.pedigree_output_name == "__void__":
# return
# else:
raise Exception(f"Can't write value '{output_v_id}': already exists.")
else:
outputs_file_name.touch()
models
special
¶
Classes¶
ModelRegistry
¶Source code in kiara/registries/models/__init__.py
class ModelRegistry(object):
_instance = None
@classmethod
def instance(cls) -> "ModelRegistry":
"""The default ModelRegistry instance.
Can be a simgleton because it only contains data that is determined by the current Python environment.
"""
if cls._instance is None:
cls._instance = ModelRegistry()
return cls._instance
def __init__(self):
self._all_models: Union[KiaraModelClassesInfo, None] = None
self._models_per_package: Dict[str, KiaraModelClassesInfo] = {}
self._sub_models: Dict[Type[KiaraModel], KiaraModelClassesInfo] = {}
@property
def all_models(self) -> KiaraModelClassesInfo:
if self._all_models is not None:
return self._all_models
self._all_models = KiaraModelClassesInfo.find_kiara_models()
return self._all_models
def get_model_cls(
self,
kiara_model_id: str,
required_subclass: Union[Type[KiaraModel], None] = None,
) -> Type[KiaraModel]:
model_info = self.all_models.item_infos.get(kiara_model_id, None)
if model_info is None:
raise Exception(
f"Can't retrieve model class for id '{kiara_model_id}': id not registered."
)
cls = model_info.python_class.get_class() # type: ignore
if required_subclass:
if not issubclass(cls, required_subclass):
raise Exception(
f"Can't retrieve sub model of '{required_subclass.__name__}' with id '{kiara_model_id}': exists, but not the required subclass."
)
return cls # type: ignore
def get_models_for_package(self, package_name: str) -> KiaraModelClassesInfo:
if package_name in self._models_per_package.keys():
return self._models_per_package[package_name]
temp = {}
for key, info in self.all_models.item_infos.items():
if info.context.labels.get("package") == package_name:
temp[key] = info
group = KiaraModelClassesInfo.construct(
group_alias=f"kiara_models.{package_name}", item_infos=temp # type: ignore
)
self._models_per_package[package_name] = group
return group
def get_models_of_type(self, model_type: Type[KiaraModel]) -> KiaraModelClassesInfo:
if model_type in self._sub_models.keys():
return self._sub_models[model_type]
sub_classes = {}
for model_id, type_info in self.all_models.item_infos.items():
cls: Type[KiaraModel] = type_info.python_class.get_class() # type: ignore
if issubclass(cls, model_type):
sub_classes[model_id] = type_info
classes = KiaraModelClassesInfo(
group_title=f"{model_type.__name__}-submodels", item_infos=sub_classes
)
self._sub_models[model_type] = classes
return classes
all_models: KiaraModelClassesInfo
property
readonly
¶get_model_cls(self, kiara_model_id, required_subclass=None)
¶Source code in kiara/registries/models/__init__.py
def get_model_cls(
self,
kiara_model_id: str,
required_subclass: Union[Type[KiaraModel], None] = None,
) -> Type[KiaraModel]:
model_info = self.all_models.item_infos.get(kiara_model_id, None)
if model_info is None:
raise Exception(
f"Can't retrieve model class for id '{kiara_model_id}': id not registered."
)
cls = model_info.python_class.get_class() # type: ignore
if required_subclass:
if not issubclass(cls, required_subclass):
raise Exception(
f"Can't retrieve sub model of '{required_subclass.__name__}' with id '{kiara_model_id}': exists, but not the required subclass."
)
return cls # type: ignore
get_models_for_package(self, package_name)
¶Source code in kiara/registries/models/__init__.py
def get_models_for_package(self, package_name: str) -> KiaraModelClassesInfo:
if package_name in self._models_per_package.keys():
return self._models_per_package[package_name]
temp = {}
for key, info in self.all_models.item_infos.items():
if info.context.labels.get("package") == package_name:
temp[key] = info
group = KiaraModelClassesInfo.construct(
group_alias=f"kiara_models.{package_name}", item_infos=temp # type: ignore
)
self._models_per_package[package_name] = group
return group
get_models_of_type(self, model_type)
¶Source code in kiara/registries/models/__init__.py
def get_models_of_type(self, model_type: Type[KiaraModel]) -> KiaraModelClassesInfo:
if model_type in self._sub_models.keys():
return self._sub_models[model_type]
sub_classes = {}
for model_id, type_info in self.all_models.item_infos.items():
cls: Type[KiaraModel] = type_info.python_class.get_class() # type: ignore
if issubclass(cls, model_type):
sub_classes[model_id] = type_info
classes = KiaraModelClassesInfo(
group_title=f"{model_type.__name__}-submodels", item_infos=sub_classes
)
self._sub_models[model_type] = classes
return classes
instance()
classmethod
¶The default ModelRegistry instance.
Can be a simgleton because it only contains data that is determined by the current Python environment.
Source code in kiara/registries/models/__init__.py
@classmethod
def instance(cls) -> "ModelRegistry":
"""The default ModelRegistry instance.
Can be a simgleton because it only contains data that is determined by the current Python environment.
"""
if cls._instance is None:
cls._instance = ModelRegistry()
return cls._instance
modules
special
¶
Base module for code that handles the import and management of [KiaraModule][kiara.module.KiaraModule] sub-classes.
logget
¶Classes¶
ModuleRegistry
¶Source code in kiara/registries/modules/__init__.py
class ModuleRegistry(object):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._cached_modules: Dict[str, Dict[CID, KiaraModule]] = {}
from kiara.utils.class_loading import find_all_kiara_modules
module_classes = find_all_kiara_modules()
self._module_classes: Mapping[str, Type[KiaraModule]] = {}
self._module_class_metadata: Dict[str, ModuleTypeInfo] = {}
for k, v in module_classes.items():
self._module_classes[k] = v
@property
def module_types(self) -> Mapping[str, Type["KiaraModule"]]:
return self._module_classes
def get_module_class(self, module_type: str) -> Type["KiaraModule"]:
cls = self._module_classes.get(module_type, None)
if cls is None:
raise ValueError(f"No module of type '{module_type}' available.")
return cls
def get_module_type_names(self) -> Iterable[str]:
return self._module_classes.keys()
def get_module_type_metadata(self, type_name: str) -> ModuleTypeInfo:
md = self._module_class_metadata.get(type_name, None)
if md is None:
md = ModuleTypeInfo.create_from_type_class(
type_cls=self.get_module_class(module_type=type_name), kiara=self._kiara
)
self._module_class_metadata[type_name] = md
return self._module_class_metadata[type_name]
def get_context_metadata(
self, alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> ModuleTypesInfo:
result = {}
for type_name in self.module_types.keys():
md = self.get_module_type_metadata(type_name=type_name)
if only_for_package:
if md.context.labels.get("package") == only_for_package:
result[type_name] = md
else:
result[type_name] = md
return ModuleTypesInfo.construct(group_alias=alias, item_infos=result) # type: ignore
def create_module(self, manifest: Union[Manifest, str]) -> "KiaraModule":
"""Create a [KiaraModule][kiara.module.KiaraModule] object from a module configuration.
Arguments:
manifest: the module configuration
"""
if isinstance(manifest, str):
manifest = Manifest.construct(module_type=manifest, module_config={})
m_cls: Type[KiaraModule] = self.get_module_class(manifest.module_type)
if not manifest.is_resolved:
resolved = m_cls._resolve_module_config(**manifest.module_config)
manifest.module_config = resolved.dict()
manifest.is_resolved = True
if self._cached_modules.setdefault(manifest.module_type, {}).get(
manifest.instance_cid, None
):
return self._cached_modules[manifest.module_type][manifest.instance_cid]
if manifest.module_type in self.get_module_type_names():
kiara_module = m_cls(module_config=manifest.module_config)
kiara_module._manifest_cache = Manifest.construct(
module_type=manifest.module_type,
module_config=manifest.module_config,
is_resolved=manifest.is_resolved,
)
else:
raise Exception(
f"Invalid module type '{manifest.module_type}'. Available type names: {', '.join(self.get_module_type_names())}"
)
return kiara_module
module_types: Mapping[str, Type[KiaraModule]]
property
readonly
¶create_module(self, manifest)
¶Create a [KiaraModule][kiara.module.KiaraModule] object from a module configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
manifest |
Union[kiara.models.module.manifest.Manifest, str] |
the module configuration |
required |
Source code in kiara/registries/modules/__init__.py
def create_module(self, manifest: Union[Manifest, str]) -> "KiaraModule":
"""Create a [KiaraModule][kiara.module.KiaraModule] object from a module configuration.
Arguments:
manifest: the module configuration
"""
if isinstance(manifest, str):
manifest = Manifest.construct(module_type=manifest, module_config={})
m_cls: Type[KiaraModule] = self.get_module_class(manifest.module_type)
if not manifest.is_resolved:
resolved = m_cls._resolve_module_config(**manifest.module_config)
manifest.module_config = resolved.dict()
manifest.is_resolved = True
if self._cached_modules.setdefault(manifest.module_type, {}).get(
manifest.instance_cid, None
):
return self._cached_modules[manifest.module_type][manifest.instance_cid]
if manifest.module_type in self.get_module_type_names():
kiara_module = m_cls(module_config=manifest.module_config)
kiara_module._manifest_cache = Manifest.construct(
module_type=manifest.module_type,
module_config=manifest.module_config,
is_resolved=manifest.is_resolved,
)
else:
raise Exception(
f"Invalid module type '{manifest.module_type}'. Available type names: {', '.join(self.get_module_type_names())}"
)
return kiara_module
get_context_metadata(self, alias=None, only_for_package=None)
¶Source code in kiara/registries/modules/__init__.py
def get_context_metadata(
self, alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> ModuleTypesInfo:
result = {}
for type_name in self.module_types.keys():
md = self.get_module_type_metadata(type_name=type_name)
if only_for_package:
if md.context.labels.get("package") == only_for_package:
result[type_name] = md
else:
result[type_name] = md
return ModuleTypesInfo.construct(group_alias=alias, item_infos=result) # type: ignore
get_module_class(self, module_type)
¶Source code in kiara/registries/modules/__init__.py
def get_module_class(self, module_type: str) -> Type["KiaraModule"]:
cls = self._module_classes.get(module_type, None)
if cls is None:
raise ValueError(f"No module of type '{module_type}' available.")
return cls
get_module_type_metadata(self, type_name)
¶Source code in kiara/registries/modules/__init__.py
def get_module_type_metadata(self, type_name: str) -> ModuleTypeInfo:
md = self._module_class_metadata.get(type_name, None)
if md is None:
md = ModuleTypeInfo.create_from_type_class(
type_cls=self.get_module_class(module_type=type_name), kiara=self._kiara
)
self._module_class_metadata[type_name] = md
return self._module_class_metadata[type_name]
get_module_type_names(self)
¶Source code in kiara/registries/modules/__init__.py
def get_module_type_names(self) -> Iterable[str]:
return self._module_classes.keys()
operations
special
¶
OP_TYPE
¶logger
¶
OperationRegistry
¶Source code in kiara/registries/operations/__init__.py
class OperationRegistry(object):
def __init__(
self,
kiara: "Kiara",
operation_type_classes: Union[Mapping[str, Type[OperationType]], None] = None,
):
self._kiara: "Kiara" = kiara
self._operation_type_classes: Union[
Dict[str, Type["OperationType"]], None
] = None
if operation_type_classes is not None:
self._operation_type_classes = dict(operation_type_classes)
self._operation_type_metadata: Dict[str, OperationTypeInfo] = {}
self._operation_types: Union[Dict[str, OperationType], None] = None
self._operations: Union[Dict[str, Operation], None] = None
self._operations_by_type: Union[Dict[str, Iterable[str]], None] = None
self._module_map: Union[Dict[str, Dict[str, Any]], None] = None
@property
def is_initialized(self) -> bool:
return self._operations is not None
def get_module_map(self) -> Mapping[str, Mapping[str, Any]]:
if not self.is_initialized:
raise Exception(
"Can't retrieve module map: operations not initialized yet."
)
if self._module_map is not None:
return self._module_map
module_map = {}
for k, v in self.operations.items():
module_map[k] = {
"module_type": v.module_type,
"module_config": v.module_config,
}
self._module_map = module_map
return self._module_map
@property
def operation_types(self) -> Mapping[str, OperationType]:
if self._operation_types is not None:
return self._operation_types
# TODO: support op type config
_operation_types = {}
for op_name, op_cls in self.operation_type_classes.items():
try:
_operation_types[op_name] = op_cls(
kiara=self._kiara, op_type_name=op_name
)
except Exception as e:
log_exception(e)
logger.debug("ignore.operation_type", operation_name=op_name, reason=e)
self._operation_types = _operation_types
return self._operation_types
def get_operation_type(self, op_type: Union[str, Type[OP_TYPE]]) -> OP_TYPE:
if not isinstance(op_type, str):
try:
op_type = op_type._operation_type_name # type: ignore
except Exception:
raise ValueError(
f"Can't retrieve operation type, invalid input type '{type(op_type)}'."
)
if op_type not in self.operation_types.keys():
raise Exception(
f"No operation type '{op_type}' registered. Available operation types: {', '.join(self.operation_types.keys())}."
)
return self.operation_types[op_type] # type: ignore
def get_type_metadata(self, type_name: str) -> OperationTypeInfo:
md = self._operation_type_metadata.get(type_name, None)
if md is None:
md = OperationTypeInfo.create_from_type_class(
kiara=self._kiara, type_cls=self.operation_type_classes[type_name]
)
self._operation_type_metadata[type_name] = md
return self._operation_type_metadata[type_name]
def get_context_metadata(
self, alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> OperationTypeClassesInfo:
result = {}
for type_name in self.operation_type_classes.keys():
md = self.get_type_metadata(type_name=type_name)
if only_for_package:
if md.context.labels.get("package") == only_for_package:
result[type_name] = md
else:
result[type_name] = md
return OperationTypeClassesInfo.construct(group_alias=alias, item_infos=result) # type: ignore
@property
def operation_type_classes(
self,
) -> Mapping[str, Type["OperationType"]]:
if self._operation_type_classes is not None:
return self._operation_type_classes
from kiara.utils.class_loading import find_all_operation_types
self._operation_type_classes = find_all_operation_types()
return self._operation_type_classes
# @property
# def operation_ids(self) -> List[str]:
# return list(self.profiles.keys())
@property
def operation_ids(self) -> Iterable[str]:
return self.operations.keys()
@property
def operations(self) -> Mapping[str, Operation]:
if self._operations is not None:
return self._operations
all_op_configs: Set[OperationConfig] = set()
for op_type in self.operation_types.values():
included_ops = op_type.retrieve_included_operation_configs()
for op in included_ops:
if isinstance(op, Mapping):
op = ManifestOperationConfig(**op)
all_op_configs.add(op)
for data_type in self._kiara.data_type_classes.values():
if hasattr(data_type, "retrieve_included_operations"):
for op in all_op_configs:
if isinstance(op, Mapping):
op = ManifestOperationConfig(**op)
all_op_configs.add(op)
operations: Dict[str, Operation] = {}
operations_by_type: Dict[str, List[str]] = {}
deferred_module_names: Dict[str, List[OperationConfig]] = {}
# first iteration
for op_config in all_op_configs:
try:
if isinstance(op_config, PipelineOperationConfig):
for mt in op_config.required_module_types:
if mt not in self._kiara.module_type_names:
deferred_module_names.setdefault(mt, []).append(op_config)
deferred_module_names.setdefault(
op_config.pipeline_name, []
).append(op_config)
continue
module_type = op_config.retrieve_module_type(kiara=self._kiara)
if module_type not in self._kiara.module_type_names:
deferred_module_names.setdefault(module_type, []).append(op_config)
else:
module_config = op_config.retrieve_module_config(kiara=self._kiara)
manifest = Manifest.construct(
module_type=module_type, module_config=module_config
)
ops = self._create_operations(manifest=manifest, doc=op_config.doc)
for op_type_name, _op in ops.items():
if _op.operation_id in operations.keys():
logger.debug(
"duplicate_operation_id",
op_id=_op.operation_id,
left_module=operations[_op.operation_id].module_type,
right_module=_op.module_type,
)
raise Exception(
f"Duplicate operation id: {_op.operation_id}"
)
operations[_op.operation_id] = _op
operations_by_type.setdefault(op_type_name, []).append(
_op.operation_id
)
except Exception as e:
details: Dict[str, Any] = {}
module_id = op_config.retrieve_module_type(kiara=self._kiara)
details["module_id"] = module_id
if module_id == "pipeline":
details["pipeline_name"] = op_config.pipeline_name # type: ignore
msg: Union[str, Exception] = str(e)
if not msg:
msg = e
details["details"] = msg
logger.error("invalid.operation", **details)
log_exception(e)
continue
error_details = {}
while deferred_module_names:
deferred_length = len(deferred_module_names)
remove_deferred_names = set()
for missing_op_id in deferred_module_names.keys():
if missing_op_id in operations.keys():
remove_deferred_names.add(missing_op_id)
continue
for op_config in deferred_module_names[missing_op_id]:
try:
if isinstance(op_config, PipelineOperationConfig):
if all(
mt in self._kiara.module_type_names
or mt in operations.keys()
for mt in op_config.required_module_types
):
module_map = {}
for mt in op_config.required_module_types:
if mt in operations.keys():
module_map[mt] = {
"module_type": operations[mt].module_type,
"module_config": operations[
mt
].module_config,
}
op_config.module_map.update(module_map)
module_config = op_config.retrieve_module_config(
kiara=self._kiara
)
manifest = Manifest.construct(
module_type="pipeline",
module_config=module_config,
)
ops = self._create_operations(
manifest=manifest,
doc=op_config.doc,
metadata=op_config.metadata,
)
else:
missing = (
mt
for mt in op_config.required_module_types
if mt not in self._kiara.module_type_names
and mt not in operations.keys()
)
raise Exception(
f"Can't find all required module types when processing pipeline '{missing_op_id}': {', '.join(missing)}"
)
else:
raise NotImplementedError(
f"Invalid type: {type(op_config)}"
)
# module_type = op_config.retrieve_module_type(kiara=self._kiara)
# module_config = op_config.retrieve_module_config(kiara=self._kiara)
#
# # TODO: merge dicts instead of update?
# new_module_config = dict(base_config)
# new_module_config.update(module_config)
#
# manifest = Manifest.construct(module_type=operation.module_type,
# module_config=new_module_config)
for op_type_name, _op in ops.items():
if _op.operation_id in operations.keys():
raise Exception(
f"Duplicate operation id: {_op.operation_id}"
)
operations[_op.operation_id] = _op
operations_by_type.setdefault(op_type_name, []).append(
_op.operation_id
)
assert _op.operation_id == op_config.pipeline_name
for _op_id in deferred_module_names.keys():
if op_config in deferred_module_names[_op_id]:
deferred_module_names[_op_id].remove(op_config)
except Exception as e:
details = {}
module_id = op_config.retrieve_module_type(kiara=self._kiara)
details["module_id"] = module_id
if module_id == "pipeline":
details["pipeline_name"] = op_config.pipeline_name # type: ignore
msg = str(e)
if not msg:
msg = e
details["details"] = msg
error_details[missing_op_id] = details
exc_info = sys.exc_info()
details["exception"] = exc_info
continue
for name, dependencies in deferred_module_names.items():
if not dependencies:
remove_deferred_names.add(name)
for rdn in remove_deferred_names:
deferred_module_names.pop(rdn)
if len(deferred_module_names) == deferred_length:
for mn in deferred_module_names:
if mn in operations.keys():
continue
details = error_details.get(missing_op_id, {"details": "-- n/a --"})
exception = details.pop("exception", None)
if exception:
log_exception(exception)
logger.error(f"invalid.operation.{mn}", operation_id=mn, **details)
break
self._operations = {}
for missing_op_id in sorted(operations.keys()):
self._operations[missing_op_id] = operations[missing_op_id]
self._operations_by_type = {}
for op_type_name in sorted(operations_by_type.keys()):
self._operations_by_type.setdefault(
op_type_name, sorted(operations_by_type[op_type_name])
)
return self._operations
def _create_operations(
self,
manifest: Manifest,
doc: Any,
metadata: Union[Mapping[str, Any], None] = None,
) -> Dict[str, Operation]:
module = self._kiara.create_module(manifest)
op_types = {}
if metadata is None:
metadata = {}
for op_name, op_type in self.operation_types.items():
op_details = op_type.check_matching_operation(module=module)
if not op_details:
continue
operation = Operation(
module_type=manifest.module_type,
module_config=manifest.module_config,
operation_id=op_details.operation_id,
operation_details=op_details,
module_details=KiaraModuleInstance.from_module(module),
metadata=metadata,
doc=doc,
)
operation._module = module
op_types[op_name] = operation
return op_types
def get_operation(self, operation_id: str) -> Operation:
if operation_id not in self.operation_ids:
raise Exception(f"No operation registered with id: {operation_id}")
op = self.operations[operation_id]
return op
def find_all_operation_types(self, operation_id: str) -> Set[str]:
result = set()
for op_type, ops in self.operations_by_type.items():
if operation_id in ops:
result.add(op_type)
return result
@property
def operations_by_type(self) -> Mapping[str, Iterable[str]]:
if self._operations_by_type is None:
self.operations # noqa
return self._operations_by_type # type: ignore
is_initialized: bool
property
readonly
¶operation_ids: Iterable[str]
property
readonly
¶operation_type_classes: Mapping[str, Type[OperationType]]
property
readonly
¶operation_types: Mapping[str, kiara.operations.OperationType]
property
readonly
¶operations: Mapping[str, kiara.models.module.operation.Operation]
property
readonly
¶operations_by_type: Mapping[str, Iterable[str]]
property
readonly
¶find_all_operation_types(self, operation_id)
¶Source code in kiara/registries/operations/__init__.py
def find_all_operation_types(self, operation_id: str) -> Set[str]:
result = set()
for op_type, ops in self.operations_by_type.items():
if operation_id in ops:
result.add(op_type)
return result
get_context_metadata(self, alias=None, only_for_package=None)
¶Source code in kiara/registries/operations/__init__.py
def get_context_metadata(
self, alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> OperationTypeClassesInfo:
result = {}
for type_name in self.operation_type_classes.keys():
md = self.get_type_metadata(type_name=type_name)
if only_for_package:
if md.context.labels.get("package") == only_for_package:
result[type_name] = md
else:
result[type_name] = md
return OperationTypeClassesInfo.construct(group_alias=alias, item_infos=result) # type: ignore
get_module_map(self)
¶Source code in kiara/registries/operations/__init__.py
def get_module_map(self) -> Mapping[str, Mapping[str, Any]]:
if not self.is_initialized:
raise Exception(
"Can't retrieve module map: operations not initialized yet."
)
if self._module_map is not None:
return self._module_map
module_map = {}
for k, v in self.operations.items():
module_map[k] = {
"module_type": v.module_type,
"module_config": v.module_config,
}
self._module_map = module_map
return self._module_map
get_operation(self, operation_id)
¶Source code in kiara/registries/operations/__init__.py
def get_operation(self, operation_id: str) -> Operation:
if operation_id not in self.operation_ids:
raise Exception(f"No operation registered with id: {operation_id}")
op = self.operations[operation_id]
return op
get_operation_type(self, op_type)
¶Source code in kiara/registries/operations/__init__.py
def get_operation_type(self, op_type: Union[str, Type[OP_TYPE]]) -> OP_TYPE:
if not isinstance(op_type, str):
try:
op_type = op_type._operation_type_name # type: ignore
except Exception:
raise ValueError(
f"Can't retrieve operation type, invalid input type '{type(op_type)}'."
)
if op_type not in self.operation_types.keys():
raise Exception(
f"No operation type '{op_type}' registered. Available operation types: {', '.join(self.operation_types.keys())}."
)
return self.operation_types[op_type] # type: ignore
get_type_metadata(self, type_name)
¶Source code in kiara/registries/operations/__init__.py
def get_type_metadata(self, type_name: str) -> OperationTypeInfo:
md = self._operation_type_metadata.get(type_name, None)
if md is None:
md = OperationTypeInfo.create_from_type_class(
kiara=self._kiara, type_cls=self.operation_type_classes[type_name]
)
self._operation_type_metadata[type_name] = md
return self._operation_type_metadata[type_name]
templates
special
¶
logger
¶Classes¶
TemplateRegistry
¶A registry collecting all the (jinja) templates that are available in the current environment.
Packages can register templates by specifying an entrypoint under 'kiara.templates', pointing to a Python module that has template files
Source code in kiara/registries/templates/__init__.py
class TemplateRegistry(object):
"""A registry collecting all the (jinja) templates that are available in the current environment.
Packages can register templates by specifying an entrypoint under 'kiara.templates', pointing to a Python module
that has template files
"""
_instance = None
@classmethod
def instance(cls) -> "TemplateRegistry":
"""The default *kiara* TemplateRegistry instance.
Can be a simgleton because it only contains data that is determined by the current Python environment.
"""
if cls._instance is None:
cls._instance = TemplateRegistry()
return cls._instance
def __init__(self):
self._template_dirs: Union[None, Mapping[str, Path]] = None
self._template_loader: Union[PrefixLoader] = None
self._environment: Union[None, Environment] = None
@property
def environment(self) -> Environment:
if self._environment is not None:
return self._environment
self._environment = Environment(
loader=self.template_loader, autoescape=select_autoescape()
)
self._environment.filters["render_model"] = partial(render_model_filter, self)
self._environment.filters["render_bool"] = boolean_filter
self._environment.filters["render_default"] = default_filter
try:
markdown = mistune.create_markdown()
except Exception:
markdown = mistune.Markdown()
self._environment.filters["markdown"] = partial(render_markdown, markdown)
return self._environment
@property
def template_dirs(self) -> Mapping[str, Path]:
if self._template_dirs is not None:
return self._template_dirs
discovered_plugins = {}
try:
import kiara_plugin # type: ignore
plugin_modules_available = True
except Exception:
plugin_modules_available = False
plugin_modules = []
if plugin_modules_available:
plugin_modules = [
name
for finder, name, ispkg in pkgutil.iter_modules(
kiara_plugin.__path__, kiara_plugin.__name__ + "." # type: ignore
)
] + [
name
for finder, name, ispkg in pkgutil.iter_modules()
if name.startswith("kiara")
]
for module_name in plugin_modules: # type: ignore
try:
module = importlib.import_module(module_name)
discovered_plugins[module_name] = module
except Exception as e:
log_exception(e)
all_template_dirs = {}
for plugin_name, module in discovered_plugins.items():
if not module.__file__:
logger.warning(
"skip.discovered_plugin", plugin_name=plugin_name, module=module
)
continue
templates_folder = os.path.join(
os.path.dirname(module.__file__), "resources", "templates"
)
if not os.path.isdir(templates_folder):
continue
all_template_dirs[plugin_name] = Path(templates_folder)
logger.debug(
"registered.templates_dir", package=plugin_name, path=templates_folder
)
self._template_dirs = all_template_dirs
return self._template_dirs
@property
def template_loader(self) -> PrefixLoader:
if self._template_loader is not None:
return self._template_loader
loaders = {}
for plugin_name, path in self.template_dirs.items():
loaders[plugin_name] = FileSystemLoader(searchpath=path)
self._template_loader = PrefixLoader(loaders)
return self._template_loader
def get_template(self, name: str) -> Template:
return self.environment.get_template(name=name)
@property
def template_names(self) -> List[str]:
"""List all available template names."""
return self.environment.list_templates()
def get_template_for_model_type(
self,
model_type: str,
template_format: str = "html",
use_generic_if_none: bool = False,
) -> Union[Template, None]:
matches = [
template_name
for template_name in self.template_names
if template_name.endswith(f"{model_type}.{template_format}")
]
if not matches and use_generic_if_none:
matches = [
template_name
for template_name in self.template_names
if template_name.endswith(f"generic_model_info.{template_format}")
]
if not matches:
return None
elif len(matches) > 1:
raise Exception(
f"Multiple templates found for model type '{model_type}' and format '{template_format}'. This is not supported yet."
)
return self.get_template(matches[0])
environment: Environment
property
readonly
¶template_dirs: Mapping[str, pathlib.Path]
property
readonly
¶template_loader: PrefixLoader
property
readonly
¶template_names: List[str]
property
readonly
¶List all available template names.
get_template(self, name)
¶Source code in kiara/registries/templates/__init__.py
def get_template(self, name: str) -> Template:
return self.environment.get_template(name=name)
get_template_for_model_type(self, model_type, template_format='html', use_generic_if_none=False)
¶Source code in kiara/registries/templates/__init__.py
def get_template_for_model_type(
self,
model_type: str,
template_format: str = "html",
use_generic_if_none: bool = False,
) -> Union[Template, None]:
matches = [
template_name
for template_name in self.template_names
if template_name.endswith(f"{model_type}.{template_format}")
]
if not matches and use_generic_if_none:
matches = [
template_name
for template_name in self.template_names
if template_name.endswith(f"generic_model_info.{template_format}")
]
if not matches:
return None
elif len(matches) > 1:
raise Exception(
f"Multiple templates found for model type '{model_type}' and format '{template_format}'. This is not supported yet."
)
return self.get_template(matches[0])
instance()
classmethod
¶The default kiara TemplateRegistry instance.
Can be a simgleton because it only contains data that is determined by the current Python environment.
Source code in kiara/registries/templates/__init__.py
@classmethod
def instance(cls) -> "TemplateRegistry":
"""The default *kiara* TemplateRegistry instance.
Can be a simgleton because it only contains data that is determined by the current Python environment.
"""
if cls._instance is None:
cls._instance = TemplateRegistry()
return cls._instance
boolean_filter(data)
¶Source code in kiara/registries/templates/__init__.py
def boolean_filter(data: bool):
return "yes" if data else "no"
default_filter(data)
¶Source code in kiara/registries/templates/__init__.py
def default_filter(data: Any):
if data in [None, SpecialValue.NO_VALUE, SpecialValue.NOT_SET]:
return ""
elif callable(data):
return str(data())
else:
return str(data)
render_markdown(markdown, markdown_str)
¶Source code in kiara/registries/templates/__init__.py
def render_markdown(markdown: mistune.Markdown, markdown_str: str):
return markdown(markdown_str)
render_model_filter(template_registry, instance)
¶Source code in kiara/registries/templates/__init__.py
def render_model_filter(template_registry: "TemplateRegistry", instance: "KiaraModel"):
template = template_registry.get_template("kiara/render/models/model_data.html")
rendered = template.render(instance=instance)
return rendered
types
special
¶
Classes¶
TypeRegistry
¶Source code in kiara/registries/types/__init__.py
class TypeRegistry(object):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._data_types: Union[bidict[str, Type[DataType]], None] = None
self._data_type_metadata: Dict[str, DataTypeClassInfo] = {}
self._cached_data_type_objects: Dict[int, DataType] = {}
# self._registered_python_classes: Dict[Type, typing.List[str]] = None # type: ignore
self._type_hierarchy: Union[nx.DiGraph, None] = None
self._lineages_cache: Dict[str, List[str]] = {}
self._type_profiles: Union[Dict[str, Mapping[str, Any]], None] = None
def invalidate_types(self):
self._data_types = None
# self._registered_python_classes = None
def retrieve_data_type(
self,
data_type_name: str,
data_type_config: Union[Mapping[str, Any], None] = None,
) -> DataType:
if data_type_config is None:
data_type_config = {}
else:
data_type_config = dict(data_type_config)
if data_type_name not in self.data_type_profiles.keys():
raise Exception(f"Data type name not registered: {data_type_name}")
data_type: str = self.data_type_profiles[data_type_name]["type_name"]
type_config = self.data_type_profiles[data_type_name]["type_config"]
if data_type_config:
type_config = dict(type_config)
type_config.update(data_type_config)
cls = self.get_data_type_cls(type_name=data_type)
hash = cls._calculate_data_type_hash(type_config)
if hash in self._cached_data_type_objects.keys():
return self._cached_data_type_objects[hash]
result = cls(**type_config)
assert result.data_type_hash == hash
self._cached_data_type_objects[result.data_type_hash] = result
return result
@property
def data_type_classes(self) -> bidict[str, Type[DataType]]:
if self._data_types is not None:
return self._data_types
self._data_types = bidict(find_all_data_types())
profiles: Dict[str, Mapping[str, Any]] = {
dn: {"type_name": dn, "type_config": {}} for dn in self._data_types.keys()
}
for name, cls in self._data_types.items():
cls_profiles = cls.retrieve_available_type_profiles()
for profile_name, type_config in cls_profiles.items():
if profile_name in profiles.keys():
raise Exception(f"Duplicate data type profile: {profile_name}")
profiles[profile_name] = {"type_name": name, "type_config": type_config}
self._type_profiles = profiles
return self._data_types
@property
def data_type_profiles(self) -> Mapping[str, Mapping[str, Any]]:
if self._type_profiles is None:
self.data_type_classes # noqa
assert self._type_profiles is not None
return self._type_profiles
@property
def data_type_hierarchy(self) -> "nx.DiGraph":
if self._type_hierarchy is not None:
return self._type_hierarchy
def recursive_base_find(cls: Type, current: Union[List[str], None] = None):
if current is None:
current = []
for base in cls.__bases__:
if base in self.data_type_classes.values():
current.append(self.data_type_classes.inverse[base])
recursive_base_find(base, current=current)
return current
bases = {}
for name, cls in self.data_type_classes.items():
bases[name] = recursive_base_find(cls)
for profile_name, details in self.data_type_profiles.items():
if not details["type_config"]:
continue
if profile_name in bases.keys():
raise Exception(
f"Invalid profile name '{profile_name}': shadowing data type. This is most likely a bug."
)
bases[profile_name] = [details["type_name"]]
import networkx as nx
hierarchy = nx.DiGraph()
hierarchy.add_node(KIARA_ROOT_TYPE_NAME)
for name, _bases in bases.items():
profile_details = self.data_type_profiles[name]
cls = self.data_type_classes[profile_details["type_name"]]
hierarchy.add_node(name, cls=cls)
if not _bases:
hierarchy.add_edge(KIARA_ROOT_TYPE_NAME, name)
else:
# we only need the first parent, all others will be taken care of by the parent of the parent
hierarchy.add_edge(_bases[0], name)
self._type_hierarchy = hierarchy
return self._type_hierarchy
def get_sub_hierarchy(self, data_type: str):
import networkx as nx
graph: nx.DiGraph = self.data_type_hierarchy
desc = nx.descendants(graph, data_type)
desc.add(data_type)
sub_graph = graph.subgraph(desc)
return sub_graph
def get_type_lineage(self, data_type_name: str) -> List[str]:
"""Returns the shortest path between the specified type and the root, in reverse direction starting from the specified type."""
if data_type_name not in self.data_type_profiles.keys():
raise Exception(f"No data type '{data_type_name}' registered.")
if data_type_name in self._lineages_cache.keys():
return self._lineages_cache[data_type_name]
import networkx as nx
path = nx.shortest_path(
self.data_type_hierarchy, KIARA_ROOT_TYPE_NAME, data_type_name
)
path.remove(KIARA_ROOT_TYPE_NAME)
self._lineages_cache[data_type_name] = list(reversed(path))
return self._lineages_cache[data_type_name]
def get_sub_types(self, data_type_name: str) -> Set[str]:
if data_type_name not in self.data_type_classes.keys():
raise Exception(f"No data type '{data_type_name}' registered.")
import networkx as nx
desc = nx.descendants(self.data_type_hierarchy, data_type_name)
return desc
def is_profile(self, data_type_name: str) -> bool:
type_config = self.data_type_profiles.get(data_type_name, {}).get(
"type_config", None
)
return True if type_config else False
def get_profile_parent(self, data_type_name: str) -> Union[None, bool]:
"""Return the parent data type of the specified data type (if that is indeed a profile name).
If the specified data type is not a profile name, 'None' will be returned.
"""
return self.data_type_profiles.get(data_type_name, {}).get("type_name", None)
def get_associated_profiles(
self, data_type_name: str
) -> Mapping[str, Mapping[str, Any]]:
if data_type_name not in self.data_type_classes.keys():
raise Exception(f"No data type '{data_type_name}' registered.")
result = {}
for profile_name, details in self.data_type_profiles.items():
if (
profile_name != data_type_name
and data_type_name == details["type_name"]
):
result[profile_name] = details
return result
@property
def data_type_names(self) -> List[str]:
return list(self.data_type_profiles.keys())
def get_data_type_cls(self, type_name: str) -> Type[DataType]:
_type_details = self.data_type_profiles.get(type_name, None)
if _type_details is None:
raise Exception(
f"No value type '{type_name}', available types: {', '.join(self.data_type_profiles.keys())}"
)
resolved_type_name: str = _type_details["type_name"]
t = self.data_type_classes.get(resolved_type_name, None)
if t is None:
raise Exception(
f"No value type '{type_name}', available types: {', '.join(self.data_type_profiles.keys())}"
)
return t
def get_data_type_instance(
self, type_name: str, type_config: Union[None, Mapping[str, Any]] = None
) -> DataType:
cls = self.get_data_type_cls(type_name=type_name)
if not type_config:
obj = cls()
else:
obj = cls(**type_config)
return obj
def get_type_metadata(self, type_name: str) -> DataTypeClassInfo:
md = self._data_type_metadata.get(type_name, None)
if md is None:
md = DataTypeClassInfo.create_from_type_class(
type_cls=self.get_data_type_cls(type_name=type_name), kiara=self._kiara
)
self._data_type_metadata[type_name] = md
return self._data_type_metadata[type_name]
def get_context_metadata(
self, alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> DataTypeClassesInfo:
result = {}
for type_name in self.data_type_classes.keys():
md = self.get_type_metadata(type_name=type_name)
if only_for_package:
if md.context.labels.get("package") == only_for_package:
result[type_name] = md
else:
result[type_name] = md
_result = DataTypeClassesInfo.construct(group_alias=alias, item_infos=result) # type: ignore
_result._kiara = self._kiara
return _result
def is_internal_type(self, data_type_name: str) -> bool:
if data_type_name not in self.data_type_profiles.keys():
return False
lineage = self.get_type_lineage(data_type_name=data_type_name)
return "any" not in lineage
data_type_classes: bidict
property
readonly
¶data_type_hierarchy: nx.DiGraph
property
readonly
¶data_type_names: List[str]
property
readonly
¶data_type_profiles: Mapping[str, Mapping[str, Any]]
property
readonly
¶get_associated_profiles(self, data_type_name)
¶Source code in kiara/registries/types/__init__.py
def get_associated_profiles(
self, data_type_name: str
) -> Mapping[str, Mapping[str, Any]]:
if data_type_name not in self.data_type_classes.keys():
raise Exception(f"No data type '{data_type_name}' registered.")
result = {}
for profile_name, details in self.data_type_profiles.items():
if (
profile_name != data_type_name
and data_type_name == details["type_name"]
):
result[profile_name] = details
return result
get_context_metadata(self, alias=None, only_for_package=None)
¶Source code in kiara/registries/types/__init__.py
def get_context_metadata(
self, alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> DataTypeClassesInfo:
result = {}
for type_name in self.data_type_classes.keys():
md = self.get_type_metadata(type_name=type_name)
if only_for_package:
if md.context.labels.get("package") == only_for_package:
result[type_name] = md
else:
result[type_name] = md
_result = DataTypeClassesInfo.construct(group_alias=alias, item_infos=result) # type: ignore
_result._kiara = self._kiara
return _result
get_data_type_cls(self, type_name)
¶Source code in kiara/registries/types/__init__.py
def get_data_type_cls(self, type_name: str) -> Type[DataType]:
_type_details = self.data_type_profiles.get(type_name, None)
if _type_details is None:
raise Exception(
f"No value type '{type_name}', available types: {', '.join(self.data_type_profiles.keys())}"
)
resolved_type_name: str = _type_details["type_name"]
t = self.data_type_classes.get(resolved_type_name, None)
if t is None:
raise Exception(
f"No value type '{type_name}', available types: {', '.join(self.data_type_profiles.keys())}"
)
return t
get_data_type_instance(self, type_name, type_config=None)
¶Source code in kiara/registries/types/__init__.py
def get_data_type_instance(
self, type_name: str, type_config: Union[None, Mapping[str, Any]] = None
) -> DataType:
cls = self.get_data_type_cls(type_name=type_name)
if not type_config:
obj = cls()
else:
obj = cls(**type_config)
return obj
get_profile_parent(self, data_type_name)
¶Return the parent data type of the specified data type (if that is indeed a profile name).
If the specified data type is not a profile name, 'None' will be returned.
Source code in kiara/registries/types/__init__.py
def get_profile_parent(self, data_type_name: str) -> Union[None, bool]:
"""Return the parent data type of the specified data type (if that is indeed a profile name).
If the specified data type is not a profile name, 'None' will be returned.
"""
return self.data_type_profiles.get(data_type_name, {}).get("type_name", None)
get_sub_hierarchy(self, data_type)
¶Source code in kiara/registries/types/__init__.py
def get_sub_hierarchy(self, data_type: str):
import networkx as nx
graph: nx.DiGraph = self.data_type_hierarchy
desc = nx.descendants(graph, data_type)
desc.add(data_type)
sub_graph = graph.subgraph(desc)
return sub_graph
get_sub_types(self, data_type_name)
¶Source code in kiara/registries/types/__init__.py
def get_sub_types(self, data_type_name: str) -> Set[str]:
if data_type_name not in self.data_type_classes.keys():
raise Exception(f"No data type '{data_type_name}' registered.")
import networkx as nx
desc = nx.descendants(self.data_type_hierarchy, data_type_name)
return desc
get_type_lineage(self, data_type_name)
¶Returns the shortest path between the specified type and the root, in reverse direction starting from the specified type.
Source code in kiara/registries/types/__init__.py
def get_type_lineage(self, data_type_name: str) -> List[str]:
"""Returns the shortest path between the specified type and the root, in reverse direction starting from the specified type."""
if data_type_name not in self.data_type_profiles.keys():
raise Exception(f"No data type '{data_type_name}' registered.")
if data_type_name in self._lineages_cache.keys():
return self._lineages_cache[data_type_name]
import networkx as nx
path = nx.shortest_path(
self.data_type_hierarchy, KIARA_ROOT_TYPE_NAME, data_type_name
)
path.remove(KIARA_ROOT_TYPE_NAME)
self._lineages_cache[data_type_name] = list(reversed(path))
return self._lineages_cache[data_type_name]
get_type_metadata(self, type_name)
¶Source code in kiara/registries/types/__init__.py
def get_type_metadata(self, type_name: str) -> DataTypeClassInfo:
md = self._data_type_metadata.get(type_name, None)
if md is None:
md = DataTypeClassInfo.create_from_type_class(
type_cls=self.get_data_type_cls(type_name=type_name), kiara=self._kiara
)
self._data_type_metadata[type_name] = md
return self._data_type_metadata[type_name]
invalidate_types(self)
¶Source code in kiara/registries/types/__init__.py
def invalidate_types(self):
self._data_types = None
# self._registered_python_classes = None
is_internal_type(self, data_type_name)
¶Source code in kiara/registries/types/__init__.py
def is_internal_type(self, data_type_name: str) -> bool:
if data_type_name not in self.data_type_profiles.keys():
return False
lineage = self.get_type_lineage(data_type_name=data_type_name)
return "any" not in lineage
is_profile(self, data_type_name)
¶Source code in kiara/registries/types/__init__.py
def is_profile(self, data_type_name: str) -> bool:
type_config = self.data_type_profiles.get(data_type_name, {}).get(
"type_config", None
)
return True if type_config else False
retrieve_data_type(self, data_type_name, data_type_config=None)
¶Source code in kiara/registries/types/__init__.py
def retrieve_data_type(
self,
data_type_name: str,
data_type_config: Union[Mapping[str, Any], None] = None,
) -> DataType:
if data_type_config is None:
data_type_config = {}
else:
data_type_config = dict(data_type_config)
if data_type_name not in self.data_type_profiles.keys():
raise Exception(f"Data type name not registered: {data_type_name}")
data_type: str = self.data_type_profiles[data_type_name]["type_name"]
type_config = self.data_type_profiles[data_type_name]["type_config"]
if data_type_config:
type_config = dict(type_config)
type_config.update(data_type_config)
cls = self.get_data_type_cls(type_name=data_type)
hash = cls._calculate_data_type_hash(type_config)
if hash in self._cached_data_type_objects.keys():
return self._cached_data_type_objects[hash]
result = cls(**type_config)
assert result.data_type_hash == hash
self._cached_data_type_objects[result.data_type_hash] = result
return result
workflows
special
¶
logger
¶Classes¶
WorkflowArchive (BaseArchive)
¶Source code in kiara/registries/workflows/__init__.py
class WorkflowArchive(BaseArchive):
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["workflow"]
@classmethod
def is_writeable(cls) -> bool:
return False
@abc.abstractmethod
def retrieve_all_workflow_aliases(self) -> Mapping[str, uuid.UUID]:
pass
@abc.abstractmethod
def retrieve_all_workflow_ids(self) -> Iterable[uuid.UUID]:
pass
@abc.abstractmethod
def retrieve_workflow_details(self, workflow_id: uuid.UUID):
pass
# @abc.abstractmethod
# def retrieve_workflow_states(
# self, workflow_id: uuid.UUID, filter: Union[WorkflowStateFilter, None] = None
# ) -> Dict[str, WorkflowState]:
# pass
@abc.abstractmethod
def retrieve_workflow_state(self, workflow_state_id: str) -> WorkflowState:
"""Retrieve workflow state details for the provided state id.
Arguments:
workflow_id: id of the workflow
workflow_state_id: the id of the workflow state
"""
@abc.abstractmethod
def retrieve_all_states_for_workflow(
self, workflow_id: uuid.UUID
) -> Mapping[str, WorkflowState]:
"""Retrieve workflow state details for the provided state id.
Arguments:
workflow_id: id of the workflow
workflow_state_id: the id of the workflow state
"""
is_writeable()
classmethod
¶Source code in kiara/registries/workflows/__init__.py
@classmethod
def is_writeable(cls) -> bool:
return False
retrieve_all_states_for_workflow(self, workflow_id)
¶Retrieve workflow state details for the provided state id.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workflow_id |
UUID |
id of the workflow |
required |
workflow_state_id |
the id of the workflow state |
required |
Source code in kiara/registries/workflows/__init__.py
@abc.abstractmethod
def retrieve_all_states_for_workflow(
self, workflow_id: uuid.UUID
) -> Mapping[str, WorkflowState]:
"""Retrieve workflow state details for the provided state id.
Arguments:
workflow_id: id of the workflow
workflow_state_id: the id of the workflow state
"""
retrieve_all_workflow_aliases(self)
¶Source code in kiara/registries/workflows/__init__.py
@abc.abstractmethod
def retrieve_all_workflow_aliases(self) -> Mapping[str, uuid.UUID]:
pass
retrieve_all_workflow_ids(self)
¶Source code in kiara/registries/workflows/__init__.py
@abc.abstractmethod
def retrieve_all_workflow_ids(self) -> Iterable[uuid.UUID]:
pass
retrieve_workflow_details(self, workflow_id)
¶Source code in kiara/registries/workflows/__init__.py
@abc.abstractmethod
def retrieve_workflow_details(self, workflow_id: uuid.UUID):
pass
retrieve_workflow_state(self, workflow_state_id)
¶Retrieve workflow state details for the provided state id.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workflow_id |
id of the workflow |
required | |
workflow_state_id |
str |
the id of the workflow state |
required |
Source code in kiara/registries/workflows/__init__.py
@abc.abstractmethod
def retrieve_workflow_state(self, workflow_state_id: str) -> WorkflowState:
"""Retrieve workflow state details for the provided state id.
Arguments:
workflow_id: id of the workflow
workflow_state_id: the id of the workflow state
"""
supported_item_types()
classmethod
¶Source code in kiara/registries/workflows/__init__.py
@classmethod
def supported_item_types(cls) -> Iterable[str]:
return ["workflow"]
WorkflowRegistry
¶Source code in kiara/registries/workflows/__init__.py
class WorkflowRegistry(object):
def __init__(self, kiara: "Kiara"):
self._kiara: Kiara = kiara
self._event_callback: Callable = self._kiara.event_registry.add_producer(self)
self._workflow_archives: Dict[str, WorkflowArchive] = {}
"""All registered archives/stores."""
self._default_alias_store: Union[str, None] = None
"""The alias of the store where new aliases are stored by default."""
self._all_aliases: Union[Dict[str, uuid.UUID], None] = None
"""All workflow aliases."""
self._all_workflow_ids: Union[Dict[uuid.UUID, str], None] = None
"""All workflow ids, with store alias as values"""
self._cached_workflows: Dict[uuid.UUID, WorkflowDetails] = {}
def register_archive(
self,
archive: WorkflowArchive,
alias: str = None,
set_as_default_store: Union[bool, None] = None,
):
workflow_archive_id = archive.archive_id
archive.register_archive(kiara=self._kiara)
if alias is None:
alias = str(workflow_archive_id)
if "." in alias:
raise Exception(
f"Can't register workflow archive with as '{alias}': registered name is not allowed to contain a '.' character (yet)."
)
if alias in self._workflow_archives.keys():
raise Exception(
f"Can't add store, workflow archive alias '{alias}' already registered."
)
self._workflow_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, WorkflowStore):
is_store = True
if set_as_default_store and self._default_alias_store is not None:
raise Exception(
f"Can't set alias store '{alias}' as default store: default store already set."
)
if self._default_alias_store is None:
is_default_store = True
self._default_alias_store = alias
event = WorkflowArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
workflow_archive_id=archive.archive_id,
workflow_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
@property
def default_alias_store(self) -> str:
if self._default_alias_store is None:
raise Exception("No default alias store set (yet).")
return self._default_alias_store
@property
def workflow_archives(self) -> Mapping[str, WorkflowArchive]:
return self._workflow_archives
def get_archive(
self, archive_id: Union[str, None] = None
) -> Union[WorkflowArchive, None]:
if archive_id is None:
archive_id = self.default_alias_store
if archive_id is None:
raise Exception("Can't retrieve default alias archive, none set (yet).")
archive = self._workflow_archives.get(archive_id, None)
return archive
@property
def workflow_aliases(self) -> Dict[str, uuid.UUID]:
"""Retrieve all registered workflow aliases."""
if self._all_aliases is not None:
return self._all_aliases
all_workflows: Dict[str, uuid.UUID] = {}
for archive_alias, archive in self._workflow_archives.items():
workflow_map = archive.retrieve_all_workflow_aliases()
for alias, w_id in workflow_map.items():
if archive_alias == self.default_alias_store:
final_alias = alias
else:
final_alias = f"{archive_alias}.{alias}"
if final_alias in all_workflows.keys():
raise Exception(
f"Inconsistent alias registry: alias '{final_alias}' available more than once."
)
all_workflows[final_alias] = w_id
self._all_aliases = all_workflows
return self._all_aliases
@property
def all_workflow_ids(self) -> Iterable[uuid.UUID]:
if self._all_workflow_ids is not None:
return self._all_workflow_ids.keys()
all_ids: Dict[uuid.UUID, str] = {}
for archive_alias, archive in self._workflow_archives.items():
ids = archive.retrieve_all_workflow_ids()
for _id in ids:
assert _id not in all_ids.keys()
all_ids[_id] = archive_alias
self._all_workflow_ids = all_ids
return self._all_workflow_ids.keys()
def get_workflow_details(self, workflow: Union[str, uuid.UUID]) -> WorkflowDetails:
if isinstance(workflow, str):
workflow_id: Union[uuid.UUID, None] = self.workflow_aliases.get(
workflow, None
)
if workflow_id is None:
try:
workflow_id = uuid.UUID(workflow)
except Exception:
pass
if workflow_id is None:
raise Exception(
f"Can't retrieve workflow with alias '{workflow}': alias not registered."
)
else:
workflow_id = workflow
if workflow_id in self._cached_workflows.keys():
return self._cached_workflows[workflow_id]
if self._all_workflow_ids is None:
self.all_workflow_ids # noqa
store_alias = self._all_workflow_ids[workflow_id] # type: ignore
store = self._workflow_archives[store_alias]
workflow_details = store.retrieve_workflow_details(workflow_id=workflow_id)
# workflow_details._kiara = self._kiara
# workflow = Workflow(kiara=self._kiara, workflow_details=workflow_details)
self._cached_workflows[workflow_id] = workflow_details
# states = store.retrieve_workflow_states(workflow_id=workflow_id)
# workflow._snapshots = states
# workflow.load_state()
return workflow_details
def register_workflow(
self,
workflow_details: Union[None, WorkflowDetails, str] = None,
workflow_aliases: Union[Iterable[str], None] = None,
) -> WorkflowDetails:
if workflow_aliases:
for workflow_alias in workflow_aliases:
if workflow_alias in self.workflow_aliases.keys():
raise Exception(
f"Can't register workflow with alias '{workflow_alias}': alias already registered."
)
store_name = self.default_alias_store
store: WorkflowStore = self.get_archive(archive_id=store_name) # type: ignore
if workflow_details is None:
workflow_details = WorkflowDetails()
workflow_details._kiara = self._kiara
elif isinstance(workflow_details, str):
workflow_details = WorkflowDetails(documentation=workflow_details) # type: ignore
workflow_details._kiara = self._kiara
# workflow_details._kiara = self._kiara
store.register_workflow(
workflow_details=workflow_details, workflow_aliases=workflow_aliases
)
# workflow = Workflow(kiara=self._kiara, workflow_details=workflow_details)
if self._all_workflow_ids is None:
self.all_workflow_ids # noqa
self._all_workflow_ids[workflow_details.workflow_id] = store_name # type: ignore
self._cached_workflows[workflow_details.workflow_id] = workflow_details
if workflow_aliases:
for workflow_alias in workflow_aliases:
self._all_workflow_ids[workflow_details.workflow_id] = store_name # type: ignore
self.workflow_aliases[workflow_alias] = workflow_details.workflow_id
# if steps:
# workflow.add_steps(*steps)
# self.save_workflow_state(workflow=workflow.workflow_id)
return workflow_details
# def get_workflow_states(
# self, workflow: Union[uuid.UUID, str]
# ) -> Dict[uuid.UUID, WorkflowState]:
#
# workflow_details = self.get_workflow_details(workflow=workflow)
# archive_alias = self._workflow_locations[workflow_details.workflow_id]
#
# archive = self.get_archive(archive_alias)
# if archive is None:
# raise Exception(
# f"Can't retrieve workflow archive '{archive_alias}', this is most likely a bug."
# )
# states = archive.retrieve_workflow_states(
# workflow_id=workflow_details.workflow_id
# )
#
# return states
def get_workflow_state(
self,
workflow_state_id: Union[str, None] = None,
workflow: Union[None, uuid.UUID, str] = None,
) -> WorkflowState:
if workflow is None and workflow_state_id is None:
raise Exception(
"Can't retrieve workflow state, neither workflow nor workflow state id specified."
)
if workflow:
workflow_details = self.get_workflow_details(workflow=workflow)
if workflow_state_id is None:
workflow_state_id = workflow_details.current_state
else:
if workflow_state_id not in workflow_details.workflow_states.values():
raise Exception(
f"Can't retrieve workflow state '{workflow_state_id}' for workflow '{workflow}': state not registered for workflow."
)
else:
raise NotImplementedError()
if workflow_state_id is None:
raise Exception(
f"Can't retrieve current workflow state, no state exists yet for workflow '{workflow}'."
)
if self._all_workflow_ids is None:
self.all_workflow_ids # noqa
archive_alias = self._all_workflow_ids[workflow_details.workflow_id] # type: ignore
archive = self.get_archive(archive_alias)
if archive is None:
raise Exception(
f"Can't retrieve workflow archive '{archive_alias}', this is most likely a bug."
)
state = archive.retrieve_workflow_state(
workflow_state_id=workflow_state_id,
)
state._kiara = self._kiara
return state
def get_all_states_for_workflow(
self, workflow: Union[uuid.UUID, str]
) -> Mapping[str, WorkflowState]:
workflow_details = self.get_workflow_details(workflow=workflow)
if self._all_workflow_ids is None:
self.all_workflow_ids # noqa
archive_alias = self._all_workflow_ids[workflow_details.workflow_id] # type: ignore
archive = self.get_archive(archive_alias)
if archive is None:
raise Exception(
f"Can't retrieve workflow archive '{archive_alias}', this is most likely a bug."
)
states = archive.retrieve_all_states_for_workflow(
workflow_id=workflow_details.workflow_id
)
return states
def add_workflow_state(
self,
workflow: Union[str, uuid.UUID, WorkflowDetails],
workflow_state: WorkflowState,
set_current: bool = True,
) -> WorkflowDetails:
# make sure the workflow is registed
created = datetime.datetime.now()
if not isinstance(workflow, WorkflowDetails):
workflow_details = self.get_workflow_details(workflow=workflow)
else:
workflow_details = workflow
workflow_details.workflow_states[created] = workflow_state.instance_id
for field_name, value_id in workflow_state.inputs.items():
self._kiara.data_registry.store_value(value=value_id)
# for field_name, value_id in workflow_state.outputs.items():
# self._kiara.data_registry.store_value(value=value_id)
store_name = self.default_alias_store
store: WorkflowStore = self.get_archive(archive_id=store_name) # type: ignore
store.add_workflow_state(workflow_state=workflow_state)
if set_current:
workflow_details.current_state = workflow_state.instance_id
store.update_workflow(workflow_details)
return workflow_details
all_workflow_ids: Iterable[uuid.UUID]
property
readonly
¶default_alias_store: str
property
readonly
¶workflow_aliases: Dict[str, uuid.UUID]
property
readonly
¶Retrieve all registered workflow aliases.
workflow_archives: Mapping[str, kiara.registries.workflows.WorkflowArchive]
property
readonly
¶add_workflow_state(self, workflow, workflow_state, set_current=True)
¶Source code in kiara/registries/workflows/__init__.py
def add_workflow_state(
self,
workflow: Union[str, uuid.UUID, WorkflowDetails],
workflow_state: WorkflowState,
set_current: bool = True,
) -> WorkflowDetails:
# make sure the workflow is registed
created = datetime.datetime.now()
if not isinstance(workflow, WorkflowDetails):
workflow_details = self.get_workflow_details(workflow=workflow)
else:
workflow_details = workflow
workflow_details.workflow_states[created] = workflow_state.instance_id
for field_name, value_id in workflow_state.inputs.items():
self._kiara.data_registry.store_value(value=value_id)
# for field_name, value_id in workflow_state.outputs.items():
# self._kiara.data_registry.store_value(value=value_id)
store_name = self.default_alias_store
store: WorkflowStore = self.get_archive(archive_id=store_name) # type: ignore
store.add_workflow_state(workflow_state=workflow_state)
if set_current:
workflow_details.current_state = workflow_state.instance_id
store.update_workflow(workflow_details)
return workflow_details
get_all_states_for_workflow(self, workflow)
¶Source code in kiara/registries/workflows/__init__.py
def get_all_states_for_workflow(
self, workflow: Union[uuid.UUID, str]
) -> Mapping[str, WorkflowState]:
workflow_details = self.get_workflow_details(workflow=workflow)
if self._all_workflow_ids is None:
self.all_workflow_ids # noqa
archive_alias = self._all_workflow_ids[workflow_details.workflow_id] # type: ignore
archive = self.get_archive(archive_alias)
if archive is None:
raise Exception(
f"Can't retrieve workflow archive '{archive_alias}', this is most likely a bug."
)
states = archive.retrieve_all_states_for_workflow(
workflow_id=workflow_details.workflow_id
)
return states
get_archive(self, archive_id=None)
¶Source code in kiara/registries/workflows/__init__.py
def get_archive(
self, archive_id: Union[str, None] = None
) -> Union[WorkflowArchive, None]:
if archive_id is None:
archive_id = self.default_alias_store
if archive_id is None:
raise Exception("Can't retrieve default alias archive, none set (yet).")
archive = self._workflow_archives.get(archive_id, None)
return archive
get_workflow_details(self, workflow)
¶Source code in kiara/registries/workflows/__init__.py
def get_workflow_details(self, workflow: Union[str, uuid.UUID]) -> WorkflowDetails:
if isinstance(workflow, str):
workflow_id: Union[uuid.UUID, None] = self.workflow_aliases.get(
workflow, None
)
if workflow_id is None:
try:
workflow_id = uuid.UUID(workflow)
except Exception:
pass
if workflow_id is None:
raise Exception(
f"Can't retrieve workflow with alias '{workflow}': alias not registered."
)
else:
workflow_id = workflow
if workflow_id in self._cached_workflows.keys():
return self._cached_workflows[workflow_id]
if self._all_workflow_ids is None:
self.all_workflow_ids # noqa
store_alias = self._all_workflow_ids[workflow_id] # type: ignore
store = self._workflow_archives[store_alias]
workflow_details = store.retrieve_workflow_details(workflow_id=workflow_id)
# workflow_details._kiara = self._kiara
# workflow = Workflow(kiara=self._kiara, workflow_details=workflow_details)
self._cached_workflows[workflow_id] = workflow_details
# states = store.retrieve_workflow_states(workflow_id=workflow_id)
# workflow._snapshots = states
# workflow.load_state()
return workflow_details
get_workflow_state(self, workflow_state_id=None, workflow=None)
¶Source code in kiara/registries/workflows/__init__.py
def get_workflow_state(
self,
workflow_state_id: Union[str, None] = None,
workflow: Union[None, uuid.UUID, str] = None,
) -> WorkflowState:
if workflow is None and workflow_state_id is None:
raise Exception(
"Can't retrieve workflow state, neither workflow nor workflow state id specified."
)
if workflow:
workflow_details = self.get_workflow_details(workflow=workflow)
if workflow_state_id is None:
workflow_state_id = workflow_details.current_state
else:
if workflow_state_id not in workflow_details.workflow_states.values():
raise Exception(
f"Can't retrieve workflow state '{workflow_state_id}' for workflow '{workflow}': state not registered for workflow."
)
else:
raise NotImplementedError()
if workflow_state_id is None:
raise Exception(
f"Can't retrieve current workflow state, no state exists yet for workflow '{workflow}'."
)
if self._all_workflow_ids is None:
self.all_workflow_ids # noqa
archive_alias = self._all_workflow_ids[workflow_details.workflow_id] # type: ignore
archive = self.get_archive(archive_alias)
if archive is None:
raise Exception(
f"Can't retrieve workflow archive '{archive_alias}', this is most likely a bug."
)
state = archive.retrieve_workflow_state(
workflow_state_id=workflow_state_id,
)
state._kiara = self._kiara
return state
register_archive(self, archive, alias=None, set_as_default_store=None)
¶Source code in kiara/registries/workflows/__init__.py
def register_archive(
self,
archive: WorkflowArchive,
alias: str = None,
set_as_default_store: Union[bool, None] = None,
):
workflow_archive_id = archive.archive_id
archive.register_archive(kiara=self._kiara)
if alias is None:
alias = str(workflow_archive_id)
if "." in alias:
raise Exception(
f"Can't register workflow archive with as '{alias}': registered name is not allowed to contain a '.' character (yet)."
)
if alias in self._workflow_archives.keys():
raise Exception(
f"Can't add store, workflow archive alias '{alias}' already registered."
)
self._workflow_archives[alias] = archive
is_store = False
is_default_store = False
if isinstance(archive, WorkflowStore):
is_store = True
if set_as_default_store and self._default_alias_store is not None:
raise Exception(
f"Can't set alias store '{alias}' as default store: default store already set."
)
if self._default_alias_store is None:
is_default_store = True
self._default_alias_store = alias
event = WorkflowArchiveAddedEvent.construct(
kiara_id=self._kiara.id,
workflow_archive_id=archive.archive_id,
workflow_archive_alias=alias,
is_store=is_store,
is_default_store=is_default_store,
)
self._event_callback(event)
register_workflow(self, workflow_details=None, workflow_aliases=None)
¶Source code in kiara/registries/workflows/__init__.py
def register_workflow(
self,
workflow_details: Union[None, WorkflowDetails, str] = None,
workflow_aliases: Union[Iterable[str], None] = None,
) -> WorkflowDetails:
if workflow_aliases:
for workflow_alias in workflow_aliases:
if workflow_alias in self.workflow_aliases.keys():
raise Exception(
f"Can't register workflow with alias '{workflow_alias}': alias already registered."
)
store_name = self.default_alias_store
store: WorkflowStore = self.get_archive(archive_id=store_name) # type: ignore
if workflow_details is None:
workflow_details = WorkflowDetails()
workflow_details._kiara = self._kiara
elif isinstance(workflow_details, str):
workflow_details = WorkflowDetails(documentation=workflow_details) # type: ignore
workflow_details._kiara = self._kiara
# workflow_details._kiara = self._kiara
store.register_workflow(
workflow_details=workflow_details, workflow_aliases=workflow_aliases
)
# workflow = Workflow(kiara=self._kiara, workflow_details=workflow_details)
if self._all_workflow_ids is None:
self.all_workflow_ids # noqa
self._all_workflow_ids[workflow_details.workflow_id] = store_name # type: ignore
self._cached_workflows[workflow_details.workflow_id] = workflow_details
if workflow_aliases:
for workflow_alias in workflow_aliases:
self._all_workflow_ids[workflow_details.workflow_id] = store_name # type: ignore
self.workflow_aliases[workflow_alias] = workflow_details.workflow_id
# if steps:
# workflow.add_steps(*steps)
# self.save_workflow_state(workflow=workflow.workflow_id)
return workflow_details
WorkflowStore (WorkflowArchive)
¶Source code in kiara/registries/workflows/__init__.py
class WorkflowStore(WorkflowArchive):
@classmethod
def is_writeable(cls) -> bool:
return True
def register_workflow(
self, workflow_details: WorkflowDetails, workflow_aliases: Iterable[str] = None
):
self._register_workflow_details(workflow_details=workflow_details)
if workflow_aliases:
if isinstance(workflow_aliases, str):
workflow_aliases = [workflow_aliases]
for workflow_alias in workflow_aliases:
self.register_alias(
workflow_id=workflow_details.workflow_id, alias=workflow_alias
)
return workflow_details
def update_workflow(self, workflow_details: WorkflowDetails):
self._update_workflow_details(workflow_details=workflow_details)
@abc.abstractmethod
def _register_workflow_details(self, workflow_details: WorkflowDetails):
pass
@abc.abstractmethod
def _update_workflow_details(self, workflow_details: WorkflowDetails):
pass
@abc.abstractmethod
def add_workflow_state(self, workflow_state: WorkflowState):
pass
@abc.abstractmethod
def register_alias(self, workflow_id: uuid.UUID, alias: str):
pass
add_workflow_state(self, workflow_state)
¶Source code in kiara/registries/workflows/__init__.py
@abc.abstractmethod
def add_workflow_state(self, workflow_state: WorkflowState):
pass
is_writeable()
classmethod
¶Source code in kiara/registries/workflows/__init__.py
@classmethod
def is_writeable(cls) -> bool:
return True
register_alias(self, workflow_id, alias)
¶Source code in kiara/registries/workflows/__init__.py
@abc.abstractmethod
def register_alias(self, workflow_id: uuid.UUID, alias: str):
pass
register_workflow(self, workflow_details, workflow_aliases=None)
¶Source code in kiara/registries/workflows/__init__.py
def register_workflow(
self, workflow_details: WorkflowDetails, workflow_aliases: Iterable[str] = None
):
self._register_workflow_details(workflow_details=workflow_details)
if workflow_aliases:
if isinstance(workflow_aliases, str):
workflow_aliases = [workflow_aliases]
for workflow_alias in workflow_aliases:
self.register_alias(
workflow_id=workflow_details.workflow_id, alias=workflow_alias
)
return workflow_details
update_workflow(self, workflow_details)
¶Source code in kiara/registries/workflows/__init__.py
def update_workflow(self, workflow_details: WorkflowDetails):
self._update_workflow_details(workflow_details=workflow_details)
Modules¶
archives
¶
FileSystemWorkflowArchive (WorkflowArchive)
¶Source code in kiara/registries/workflows/archives.py
class FileSystemWorkflowArchive(WorkflowArchive):
_archive_type_name = "filesystem_workflow_archive"
_config_cls = FileSystemArchiveConfig
def __init__(self, archive_id: uuid.UUID, config: ARCHIVE_CONFIG_CLS):
super().__init__(archive_id=archive_id, config=config)
self._base_path: Union[Path, None] = None
self.alias_store_path.mkdir(parents=True, exist_ok=True)
@property
def workflow_store_path(self) -> Path:
if self._base_path is not None:
return self._base_path
self._base_path = Path(self.config.archive_path).absolute() # type: ignore
self._base_path.mkdir(parents=True, exist_ok=True)
return self._base_path
@property
def alias_store_path(self) -> Path:
return self.workflow_store_path / "aliases"
def _delete_archive(self):
shutil.rmtree(self.workflow_store_path)
@property
def workflow_path(self) -> Path:
return self.workflow_store_path / "workflows"
@property
def workflow_states_path(self) -> Path:
return self.workflow_store_path / "states"
def get_workflow_details_path(self, workflow_id: uuid.UUID) -> Path:
return self.workflow_path / str(workflow_id) / "workflow.json"
def get_alias_path(self, alias: str):
return self.alias_store_path / f"{alias}.alias"
def retrieve_all_workflow_aliases(self) -> Mapping[str, uuid.UUID]:
all_aliases = self.alias_store_path.glob("*.alias")
result: Dict[str, uuid.UUID] = {}
for path in all_aliases:
alias = path.name[0:-6]
workflow_path = path.resolve()
workflow_id = uuid.UUID(workflow_path.parent.name)
if alias in result.keys():
raise Exception(
f"Invalid internal state for workflow archive '{self.archive_id}': duplicate alias '{alias}'."
)
result[alias] = workflow_id
return result
def retrieve_all_workflow_ids(self) -> Iterable[uuid.UUID]:
all_ids = self.workflow_path.glob("*")
result = []
for path in all_ids:
workflow_id = uuid.UUID(path.name)
result.append(workflow_id)
return result
def retrieve_workflow_details(self, workflow_id: uuid.UUID) -> WorkflowDetails:
workflow_path = self.get_workflow_details_path(workflow_id=workflow_id)
if not workflow_path.exists():
raise Exception(
f"Can't retrieve workflow with id '{workflow_id}': id does not exist."
)
workflow_json = workflow_path.read_text()
workflow_data = orjson.loads(workflow_json)
workflow = WorkflowDetails(**workflow_data)
workflow._kiara = self.kiara_context
return workflow
def retrieve_workflow_state(self, workflow_state_id: str) -> WorkflowState:
workflow_state_path = self.workflow_states_path / f"{workflow_state_id}.state"
if not workflow_state_path.exists():
raise Exception(f"No workflow state with id '{workflow_state_id}' exists.")
_data = workflow_state_path.read_text()
_json = orjson.loads(_data)
_json["pipeline_info"]["pipeline_structure"] = {
"pipeline_config": _json["pipeline_info"]["pipeline_structure"][
"pipeline_config"
]
}
_state = WorkflowState(**_json)
_state.pipeline_info._kiara = self.kiara_context
_state._kiara = self.kiara_context
return _state
def retrieve_all_states_for_workflow(
self, workflow_id: uuid.UUID
) -> Mapping[str, WorkflowState]:
details = self.retrieve_workflow_details(workflow_id=workflow_id)
result = {}
for ws_id in details.workflow_states.values():
ws_state = self.retrieve_workflow_state(workflow_state_id=ws_id)
result[ws_id] = ws_state
return result
alias_store_path: Path
property
readonly
¶workflow_path: Path
property
readonly
¶workflow_states_path: Path
property
readonly
¶workflow_store_path: Path
property
readonly
¶
_config_cls (ArchiveConfig)
private
pydantic-model
¶Source code in kiara/registries/workflows/archives.py
class FileSystemArchiveConfig(ArchiveConfig):
archive_path: str = Field(
description="The path where the data for this archive is stored."
)
get_alias_path(self, alias)
¶Source code in kiara/registries/workflows/archives.py
def get_alias_path(self, alias: str):
return self.alias_store_path / f"{alias}.alias"
get_workflow_details_path(self, workflow_id)
¶Source code in kiara/registries/workflows/archives.py
def get_workflow_details_path(self, workflow_id: uuid.UUID) -> Path:
return self.workflow_path / str(workflow_id) / "workflow.json"
retrieve_all_states_for_workflow(self, workflow_id)
¶Retrieve workflow state details for the provided state id.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workflow_id |
UUID |
id of the workflow |
required |
workflow_state_id |
the id of the workflow state |
required |
Source code in kiara/registries/workflows/archives.py
def retrieve_all_states_for_workflow(
self, workflow_id: uuid.UUID
) -> Mapping[str, WorkflowState]:
details = self.retrieve_workflow_details(workflow_id=workflow_id)
result = {}
for ws_id in details.workflow_states.values():
ws_state = self.retrieve_workflow_state(workflow_state_id=ws_id)
result[ws_id] = ws_state
return result
retrieve_all_workflow_aliases(self)
¶Source code in kiara/registries/workflows/archives.py
def retrieve_all_workflow_aliases(self) -> Mapping[str, uuid.UUID]:
all_aliases = self.alias_store_path.glob("*.alias")
result: Dict[str, uuid.UUID] = {}
for path in all_aliases:
alias = path.name[0:-6]
workflow_path = path.resolve()
workflow_id = uuid.UUID(workflow_path.parent.name)
if alias in result.keys():
raise Exception(
f"Invalid internal state for workflow archive '{self.archive_id}': duplicate alias '{alias}'."
)
result[alias] = workflow_id
return result
retrieve_all_workflow_ids(self)
¶Source code in kiara/registries/workflows/archives.py
def retrieve_all_workflow_ids(self) -> Iterable[uuid.UUID]:
all_ids = self.workflow_path.glob("*")
result = []
for path in all_ids:
workflow_id = uuid.UUID(path.name)
result.append(workflow_id)
return result
retrieve_workflow_details(self, workflow_id)
¶Source code in kiara/registries/workflows/archives.py
def retrieve_workflow_details(self, workflow_id: uuid.UUID) -> WorkflowDetails:
workflow_path = self.get_workflow_details_path(workflow_id=workflow_id)
if not workflow_path.exists():
raise Exception(
f"Can't retrieve workflow with id '{workflow_id}': id does not exist."
)
workflow_json = workflow_path.read_text()
workflow_data = orjson.loads(workflow_json)
workflow = WorkflowDetails(**workflow_data)
workflow._kiara = self.kiara_context
return workflow
retrieve_workflow_state(self, workflow_state_id)
¶Retrieve workflow state details for the provided state id.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workflow_id |
id of the workflow |
required | |
workflow_state_id |
str |
the id of the workflow state |
required |
Source code in kiara/registries/workflows/archives.py
def retrieve_workflow_state(self, workflow_state_id: str) -> WorkflowState:
workflow_state_path = self.workflow_states_path / f"{workflow_state_id}.state"
if not workflow_state_path.exists():
raise Exception(f"No workflow state with id '{workflow_state_id}' exists.")
_data = workflow_state_path.read_text()
_json = orjson.loads(_data)
_json["pipeline_info"]["pipeline_structure"] = {
"pipeline_config": _json["pipeline_info"]["pipeline_structure"][
"pipeline_config"
]
}
_state = WorkflowState(**_json)
_state.pipeline_info._kiara = self.kiara_context
_state._kiara = self.kiara_context
return _state
FileSystemWorkflowStore (FileSystemWorkflowArchive, WorkflowStore)
¶Source code in kiara/registries/workflows/archives.py
class FileSystemWorkflowStore(FileSystemWorkflowArchive, WorkflowStore):
_archive_type_name = "filesystem_workflow_store"
def _register_workflow_details(self, workflow_details: WorkflowDetails):
workflow_path = self.get_workflow_details_path(
workflow_id=workflow_details.workflow_id
)
if workflow_path.exists():
raise Exception(
f"Can't register workflow with id '{workflow_details.workflow_id}': id already registered."
)
workflow_path.parent.mkdir(parents=True, exist_ok=False)
workflow_json = workflow_details.json()
workflow_path.write_text(workflow_json)
def _update_workflow_details(self, workflow_details: WorkflowDetails):
workflow_path = self.get_workflow_details_path(
workflow_id=workflow_details.workflow_id
)
if not workflow_path.exists():
raise Exception(
f"Can't update workflow with id '{workflow_details.workflow_id}': id not registered."
)
workflow_json = workflow_details.json(option=orjson.OPT_NON_STR_KEYS)
workflow_path.write_text(workflow_json)
def register_alias(self, workflow_id: uuid.UUID, alias: str, force: bool = False):
alias_path = self.get_alias_path(alias=alias)
if not force and alias_path.exists():
raise Exception(
f"Can't register workflow alias '{alias}': alias already registered."
)
elif alias_path.exists():
alias_path.unlink()
workflow_path = self.get_workflow_details_path(workflow_id=workflow_id)
if not workflow_path.exists():
raise Exception(
f"Can't register workflow alias '{alias}': target id '{workflow_id}' not registered."
)
alias_path.symlink_to(workflow_path)
def add_workflow_state(self, workflow_state: WorkflowState):
self.workflow_states_path.mkdir(exist_ok=True, parents=True)
workflow_state_path = (
self.workflow_states_path / f"{workflow_state.instance_id}.state"
)
workflow_state_json = workflow_state.json()
workflow_state_path.write_text(workflow_state_json)
add_workflow_state(self, workflow_state)
¶Source code in kiara/registries/workflows/archives.py
def add_workflow_state(self, workflow_state: WorkflowState):
self.workflow_states_path.mkdir(exist_ok=True, parents=True)
workflow_state_path = (
self.workflow_states_path / f"{workflow_state.instance_id}.state"
)
workflow_state_json = workflow_state.json()
workflow_state_path.write_text(workflow_state_json)
register_alias(self, workflow_id, alias, force=False)
¶Source code in kiara/registries/workflows/archives.py
def register_alias(self, workflow_id: uuid.UUID, alias: str, force: bool = False):
alias_path = self.get_alias_path(alias=alias)
if not force and alias_path.exists():
raise Exception(
f"Can't register workflow alias '{alias}': alias already registered."
)
elif alias_path.exists():
alias_path.unlink()
workflow_path = self.get_workflow_details_path(workflow_id=workflow_id)
if not workflow_path.exists():
raise Exception(
f"Can't register workflow alias '{alias}': target id '{workflow_id}' not registered."
)
alias_path.symlink_to(workflow_path)
render
special
¶
Modules¶
pipeline
¶
RENDER_CONFIG
¶RENDER_SOURCE_TYPE
¶Classes¶
JinjaPipelineRenderConfig (KiaraRendererConfig)
pydantic-model
¶Source code in kiara/render/pipeline.py
class JinjaPipelineRenderConfig(KiaraRendererConfig):
template: str = Field(
description="The template to use to render the pipeline. Either a path to a template file, or the template string directly."
)
JinjaPipelineRenderer (KiaraRenderer)
¶Source code in kiara/render/pipeline.py
class JinjaPipelineRenderer(KiaraRenderer[Pipeline, JinjaPipelineRenderConfig]):
_render_config_cls = JinjaPipelineRenderConfig
def get_render_source_type(self) -> Type[Pipeline]:
return Pipeline
def _render_object(self, object: Pipeline) -> Any:
template = self.config.template
if os.path.isfile(template):
path = Path(template)
loader = FileSystemLoader(path.parent)
env: Environment = Environment(loader=loader)
env.filters["extract_raw_data"] = partial(extract_raw_value, self._kiara)
_template = env.get_template(path.name)
else:
env = Environment(loader=BaseLoader())
_template = env.from_string(template)
rendered = _template.render(pipeline=object, config=self.config)
return rendered
def _augment_inputs(self, **inputs: Any) -> Mapping[str, Any]:
# pipeline_input = inputs.get("inputs", None)
module = inputs.get("module")
module_config = inputs.get("module_config", None)
module_obj: "PipelineModule" = self._kiara.create_module( # type: ignore
module_type=module, module_config=module_config # type: ignore
)
if not module_obj.is_pipeline():
raise Exception("Only pipeline modules supported (for now).")
step_inputs: Dict[str, Dict[str, Any]] = {}
# for k, v in pipeline_input.items():
# pi = module_obj.structure.pipeline_inputs.get(k)
# assert pi
# if len(pi.connected_inputs) != 1:
# raise NotImplementedError()
#
# ci = pi.connected_inputs[0]
# if isinstance(v, str):
# v = f'"{v}"'
# step_inputs.setdefault(ci.step_id, {})[ci.value_name] = v
result = {"structure": module_obj.config.structure, "input_values": step_inputs}
if "template" in inputs.keys():
template = inputs["template"]
else:
template = "notebook"
if template in ["notebook", "python-script"]:
template = os.path.join(
KIARA_RESOURCES_FOLDER, "templates", f"{template}.j2"
)
result["template"] = template
return result
def _post_process(self, rendered: Any) -> Any:
is_notebook = True
if is_notebook:
import jupytext
notebook = jupytext.reads(rendered, fmt="py:percent")
converted = jupytext.writes(notebook, fmt="notebook")
return converted
else:
import black
from black import Mode
cleaned = black.format_str(rendered, mode=Mode())
return cleaned
get_render_source_type(self)
¶Source code in kiara/render/pipeline.py
def get_render_source_type(self) -> Type[Pipeline]:
return Pipeline
KiaraRenderer (ABC, Generic)
¶Source code in kiara/render/pipeline.py
class KiaraRenderer(abc.ABC, Generic[RENDER_SOURCE_TYPE, RENDER_CONFIG]):
_render_config_cls: Type[RENDER_CONFIG] = KiaraRendererConfig # type: ignore
def __init__(
self,
config: Union[None, Mapping[str, Any], KiaraRendererConfig] = None,
kiara: Union[None, "Kiara"] = None,
):
if kiara is None:
from kiara.context import Kiara
kiara = Kiara.instance()
self._kiara: "Kiara" = kiara
if config is None:
self._config: RENDER_CONFIG = self.__class__._render_config_cls()
elif isinstance(config, Mapping):
self._config = self.__class__._render_config_cls(**config)
elif not isinstance(config, self.__class__._render_config_cls):
raise Exception(
f"Can't create renderer instance, invalid config type: {type(config)}, must be: {self.__class__._render_config_cls.__name__}"
)
else:
self._config = config
@property
def config(self) -> RENDER_CONFIG:
return self._config
@abc.abstractmethod
def get_render_source_type(self) -> Type[RENDER_SOURCE_TYPE]:
pass
@abc.abstractmethod
def _render_object(self, object: RENDER_SOURCE_TYPE) -> Any:
pass
def _post_process(self, rendered: Any) -> Any:
return rendered
def render(self, object: RENDER_SOURCE_TYPE):
rendered = self._render_object(object=object)
post_processed = self._post_process(rendered=rendered)
return post_processed
config: ~RENDER_CONFIG
property
readonly
¶get_render_source_type(self)
¶Source code in kiara/render/pipeline.py
@abc.abstractmethod
def get_render_source_type(self) -> Type[RENDER_SOURCE_TYPE]:
pass
render(self, object)
¶Source code in kiara/render/pipeline.py
def render(self, object: RENDER_SOURCE_TYPE):
rendered = self._render_object(object=object)
post_processed = self._post_process(rendered=rendered)
return post_processed
KiaraRendererConfig (KiaraModel)
pydantic-model
¶Source code in kiara/render/pipeline.py
class KiaraRendererConfig(KiaraModel):
pass
utils
special
¶
CAMEL_TO_SNAKE_REGEX
¶
SUBCLASS_TYPE
¶
WORD_REGEX_PATTERN
¶
logger
¶
Functions¶
camel_case_to_snake_case(camel_text, repl='_')
¶
Source code in kiara/utils/__init__.py
def camel_case_to_snake_case(camel_text: str, repl: str = "_"):
return CAMEL_TO_SNAKE_REGEX.sub(repl, camel_text).lower()
check_valid_field_names(*field_names)
¶
Check whether the provided field names are all valid.
Returns:
| Type | Description |
|---|---|
List[str] |
an iterable of strings with invalid field names |
Source code in kiara/utils/__init__.py
def check_valid_field_names(*field_names) -> List[str]:
"""Check whether the provided field names are all valid.
Returns:
an iterable of strings with invalid field names
"""
return [x for x in field_names if x in INVALID_VALUE_NAMES or x.startswith("_")]
find_free_id(stem, current_ids, sep='_')
¶
Find a free var (or other name) based on a stem string, based on a list of provided existing names.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stem |
str |
the base string to use |
required |
current_ids |
Iterable[str] |
currently existing names |
required |
method |
str |
the method to create new names (allowed: 'count' -- for now) |
required |
method_args |
dict |
prototing_config for the creation method |
required |
Returns:
| Type | Description |
|---|---|
str |
a free name |
Source code in kiara/utils/__init__.py
def find_free_id(
stem: str,
current_ids: Iterable[str],
sep="_",
) -> str:
"""Find a free var (or other name) based on a stem string, based on a list of provided existing names.
Args:
stem (str): the base string to use
current_ids (Iterable[str]): currently existing names
method (str): the method to create new names (allowed: 'count' -- for now)
method_args (dict): prototing_config for the creation method
Returns:
str: a free name
"""
start_count = 1
if stem not in current_ids:
return stem
i = start_count
# new_name = None
while True:
new_name = f"{stem}{sep}{i}"
if new_name in current_ids:
i = i + 1
continue
break
return new_name
first_line(text)
¶
Source code in kiara/utils/__init__.py
def first_line(text: str):
if "\n" in text:
return text.split("\n")[0].strip()
else:
return text
get_auto_workflow_alias(module_type, use_incremental_ids=False)
¶
Return an id for a workflow obj of a provided module class.
If 'use_incremental_ids' is set to True, a unique id is returned.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
module_type |
str |
the name of the module type |
required |
use_incremental_ids |
bool |
whether to return a unique (incremental) id |
False |
Returns:
| Type | Description |
|---|---|
str |
a module id |
Source code in kiara/utils/__init__.py
def get_auto_workflow_alias(module_type: str, use_incremental_ids: bool = False) -> str:
"""Return an id for a workflow obj of a provided module class.
If 'use_incremental_ids' is set to True, a unique id is returned.
Args:
module_type (str): the name of the module type
use_incremental_ids (bool): whether to return a unique (incremental) id
Returns:
str: a module id
"""
if not use_incremental_ids:
return module_type
nr = _AUTO_MODULE_ID.setdefault(module_type, 0)
_AUTO_MODULE_ID[module_type] = nr + 1
return f"{module_type}_{nr}"
get_dev_config()
¶
Source code in kiara/utils/__init__.py
def get_dev_config() -> "KiaraDevSettings":
from kiara.utils.develop import KIARA_DEV_SETTINGS
return KIARA_DEV_SETTINGS
is_debug()
¶
Source code in kiara/utils/__init__.py
def is_debug() -> bool:
debug = os.environ.get("DEBUG", "")
if debug.lower() == "true":
return True
else:
return False
is_develop()
¶
Source code in kiara/utils/__init__.py
def is_develop() -> bool:
develop = os.environ.get("DEVELOP", "")
if not develop:
develop = os.environ.get("DEV", "")
if develop and develop.lower() != "false":
return True
return False
is_jupyter()
¶
Source code in kiara/utils/__init__.py
def is_jupyter() -> bool:
try:
get_ipython # type: ignore
except NameError:
return False
ipython = get_ipython() # type: ignore # noqa
shell = ipython.__class__.__name__
if shell == "TerminalInteractiveShell":
return False
elif "google.colab" in str(ipython.__class__) or shell == "ZMQInteractiveShell":
return True
else:
return False
log_exception(exc)
¶
Source code in kiara/utils/__init__.py
def log_exception(exc: Exception):
if is_debug():
logger.exception(exc)
if is_develop():
from kiara.utils.develop import DetailLevel
config = get_dev_config()
if config.log.exc in [DetailLevel.NONE, "none"]:
return
show_locals = config.log.exc in [DetailLevel.FULL, "full"]
from kiara.interfaces import get_console
from kiara.utils.develop import log_dev_message
exc_info = sys.exc_info()
if not exc_info:
# TODO: create exc_info from exception?
if not is_debug():
logger.exception(exc)
else:
console = get_console()
log_dev_message(
Traceback.from_exception(
*exc_info, show_locals=show_locals, width=console.width - 4 # type: ignore
),
title="Exception details",
)
log_message(msg, **data)
¶
Source code in kiara/utils/__init__.py
def log_message(msg: str, **data):
if is_debug():
logger.debug(msg, **data)
# else:
# logger.debug(msg, **data)
to_camel_case(text)
¶
Source code in kiara/utils/__init__.py
def to_camel_case(text: str) -> str:
words = WORD_REGEX_PATTERN.split(text)
return "".join(w.title() for i, w in enumerate(words))
Modules¶
class_loading
¶
KiaraEntryPointItem
¶KiaraEntryPointIterable
¶SUBCLASS_TYPE
¶logger
¶Functions¶
find_all_archive_types()
¶Find all KiaraArchive subclasses via package entry points.
Source code in kiara/utils/class_loading.py
def find_all_archive_types() -> Dict[str, Type["KiaraArchive"]]:
"""Find all [KiaraArchive][kiara.registries.KiaraArchive] subclasses via package entry points."""
from kiara.registries import KiaraArchive
return load_all_subclasses_for_entry_point(
entry_point_name="kiara.archive_type",
base_class=KiaraArchive, # type: ignore
type_id_key="_archive_type_name",
type_id_func=_cls_name_id_func,
attach_python_metadata=False,
)
find_all_cli_subcommands()
¶Source code in kiara/utils/class_loading.py
def find_all_cli_subcommands():
entry_point_name = "kiara.cli_subcommands"
log2 = logging.getLogger("stevedore")
out_hdlr = logging.StreamHandler(sys.stdout)
out_hdlr.setFormatter(
logging.Formatter(
f"{entry_point_name} plugin search message/error -> %(message)s"
)
)
out_hdlr.setLevel(logging.INFO)
log2.addHandler(out_hdlr)
if is_debug():
log2.setLevel(logging.DEBUG)
else:
out_hdlr.setLevel(logging.INFO)
log2.setLevel(logging.INFO)
log_message("events.loading.entry_points", entry_point_name=entry_point_name)
mgr = ExtensionManager(
namespace=entry_point_name,
invoke_on_load=False,
propagate_map_exceptions=True,
)
return [plugin.plugin for plugin in mgr]
find_all_data_types()
¶Find all [KiaraModule][kiara.module.KiaraModule] subclasses via package entry points.
TODO
Source code in kiara/utils/class_loading.py
def find_all_data_types() -> Dict[str, Type["DataType"]]:
"""Find all [KiaraModule][kiara.module.KiaraModule] subclasses via package entry points.
TODO
"""
from kiara.data_types import DataType
all_data_types = load_all_subclasses_for_entry_point(
entry_point_name="kiara.data_types",
base_class=DataType, # type: ignore
type_id_key="_data_type_name",
type_id_func=_cls_name_id_func,
)
invalid = [x for x in all_data_types.keys() if "." in x]
if invalid:
raise Exception(
f"Invalid value type name(s), type names can't contain '.': {', '.join(invalid)}"
)
return all_data_types
find_all_kiara_model_classes()
¶Find all [KiaraModule][kiara.module.KiaraModule] subclasses via package entry points.
TODO
Source code in kiara/utils/class_loading.py
def find_all_kiara_model_classes() -> Dict[str, Type["KiaraModel"]]:
"""Find all [KiaraModule][kiara.module.KiaraModule] subclasses via package entry points.
TODO
"""
from kiara.models import KiaraModel
return load_all_subclasses_for_entry_point(
entry_point_name="kiara.model_classes",
base_class=KiaraModel, # type: ignore
type_id_key="_kiara_model_id",
type_id_func=_cls_name_id_func,
attach_python_metadata=False,
)
find_all_kiara_modules()
¶Find all [KiaraModule][kiara.module.KiaraModule] subclasses via package entry points.
TODO
Source code in kiara/utils/class_loading.py
def find_all_kiara_modules() -> Dict[str, Type["KiaraModule"]]:
"""Find all [KiaraModule][kiara.module.KiaraModule] subclasses via package entry points.
TODO
"""
from kiara.modules import KiaraModule
modules = load_all_subclasses_for_entry_point(
entry_point_name="kiara.modules",
base_class=KiaraModule, # type: ignore
type_id_key="_module_type_name",
attach_python_metadata=True,
)
result = {}
# need to test this, since I couldn't add an abstract method to the KiaraModule class itself (mypy complained because it is potentially overloaded)
for k, cls in modules.items():
if not hasattr(cls, "process"):
if is_develop():
msg = f"Invalid kiara module: **{cls.__module__}.{cls.__name__}**\n\nMissing method(s):\n- *process*"
log_dev_message(msg=Markdown(msg))
# TODO: check signature of process method
log_message(
"ignore.subclass",
sub_class=cls,
base_class=KiaraModule,
reason="'process' method is missing",
)
continue
result[k] = cls
return result
find_all_kiara_pipeline_paths(skip_errors=False)
¶Source code in kiara/utils/class_loading.py
def find_all_kiara_pipeline_paths(
skip_errors: bool = False,
) -> Dict[str, Union[Mapping[str, Any], None]]:
import logging
log2 = logging.getLogger("stevedore")
out_hdlr = logging.StreamHandler(sys.stdout)
out_hdlr.setFormatter(
logging.Formatter("kiara pipeline search plugin error -> %(message)s")
)
out_hdlr.setLevel(logging.INFO)
log2.addHandler(out_hdlr)
log2.setLevel(logging.INFO)
log_message("events.loading.pipelines")
mgr = ExtensionManager(
namespace="kiara.pipelines", invoke_on_load=False, propagate_map_exceptions=True
)
paths: Dict[str, Union[Mapping[str, Any], None]] = {}
# TODO: make sure we load 'core' first?
for plugin in mgr:
name = plugin.name
if (
isinstance(plugin.plugin, tuple)
and len(plugin.plugin) >= 1
and callable(plugin.plugin[0])
) or callable(plugin.plugin):
try:
if callable(plugin.plugin):
func = plugin.plugin
args = []
else:
func = plugin.plugin[0]
args = plugin.plugin[1:]
f_args = []
metadata: Union[Mapping[str, Any], None] = None
if len(args) >= 1:
f_args.append(args[0])
if len(args) >= 2:
metadata = args[1]
assert isinstance(metadata, Mapping)
if len(args) > 3:
logger.debug(
"ignore.pipeline_lookup_arguments",
reason="more than 2 arguments provided",
surplus_args=args[2:],
path=f_args[0],
)
result = func(f_args[0])
if not result:
continue
if isinstance(result, str):
paths[result] = metadata
else:
for path in paths:
assert path not in paths.keys()
paths[path] = metadata
except Exception as e:
log_exception(e)
if skip_errors:
log_message(
"ignore.pipline_entrypoint", entrypoint_name=name, reason=str(e)
)
continue
raise Exception(f"Error trying to load plugin '{plugin.plugin}': {e}")
else:
if skip_errors:
log_message(
"ignore.pipline_entrypoint",
entrypoint_name=name,
reason=f"invalid plugin type '{type(plugin.plugin)}'",
)
continue
msg = f"Can't load pipelines for entrypoint '{name}': invalid plugin type '{type(plugin.plugin)}'"
raise Exception(msg)
return paths
find_all_operation_types()
¶Source code in kiara/utils/class_loading.py
def find_all_operation_types() -> Dict[str, Type["OperationType"]]:
from kiara.operations import OperationType
result = load_all_subclasses_for_entry_point(
entry_point_name="kiara.operation_types",
base_class=OperationType, # type: ignore
type_id_key="_operation_type_name",
)
return result
find_data_types_under(module)
¶Source code in kiara/utils/class_loading.py
def find_data_types_under(module: Union[str, ModuleType]) -> List[Type["DataType"]]:
from kiara.data_types import DataType
return find_subclasses_under(
base_class=DataType, # type: ignore
python_module=module,
)
find_kiara_model_classes_under(module)
¶Source code in kiara/utils/class_loading.py
def find_kiara_model_classes_under(
module: Union[str, ModuleType]
) -> List[Type["KiaraModel"]]:
from kiara.models import KiaraModel
result = find_subclasses_under(
base_class=KiaraModel, # type: ignore
python_module=module,
)
return result
find_kiara_modules_under(module)
¶Source code in kiara/utils/class_loading.py
def find_kiara_modules_under(
module: Union[str, ModuleType],
) -> List[Type["KiaraModule"]]:
from kiara.modules import KiaraModule
return find_subclasses_under(
base_class=KiaraModule, # type: ignore
python_module=module,
)
find_operations_under(module)
¶Source code in kiara/utils/class_loading.py
def find_operations_under(
module: Union[str, ModuleType]
) -> List[Type["OperationType"]]:
from kiara.operations import OperationType
return find_subclasses_under(
base_class=OperationType, # type: ignore
python_module=module,
)
find_pipeline_base_path_for_module(module)
¶Source code in kiara/utils/class_loading.py
def find_pipeline_base_path_for_module(
module: Union[str, ModuleType]
) -> Union[str, None]:
if hasattr(sys, "frozen"):
raise NotImplementedError("Pyinstaller bundling not supported yet.")
if isinstance(module, str):
module = importlib.import_module(module)
module_file = module.__file__
assert module_file is not None
path = os.path.dirname(module_file)
if not os.path.exists:
log_message("ignore.pipeline_folder", path=path, reason="folder does not exist")
return None
return path
find_subclasses_under(base_class, python_module)
¶Find all (non-abstract) subclasses of a base class that live under a module (recursively).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
base_class |
Type[~SUBCLASS_TYPE] |
the parent class |
required |
python_module |
Union[str, module] |
the Python module to search |
required |
Returns:
| Type | Description |
|---|---|
List[Type[~SUBCLASS_TYPE]] |
a list of all subclasses |
Source code in kiara/utils/class_loading.py
def find_subclasses_under(
base_class: Type[SUBCLASS_TYPE],
python_module: Union[str, ModuleType],
) -> List[Type[SUBCLASS_TYPE]]:
"""Find all (non-abstract) subclasses of a base class that live under a module (recursively).
Arguments:
base_class: the parent class
python_module: the Python module to search
Returns:
a list of all subclasses
"""
if hasattr(sys, "frozen"):
raise NotImplementedError("Pyinstaller bundling not supported yet.")
try:
if isinstance(python_module, str):
python_module = importlib.import_module(python_module)
_import_modules_recursively(python_module)
except Exception as e:
log_exception(e)
log_message("ignore.python_module", module=str(python_module), reason=str(e))
return []
subclasses: Iterable[Type[SUBCLASS_TYPE]] = _get_all_subclasses(base_class)
result = []
for sc in subclasses:
if not sc.__module__.startswith(python_module.__name__):
continue
result.append(sc)
return result
load_all_subclasses_for_entry_point(entry_point_name, base_class, ignore_abstract_classes=True, type_id_key=None, type_id_func=None, type_id_no_attach=False, attach_python_metadata=False)
¶Find all subclasses of a base class via package entry points.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
entry_point_name |
str |
the entry point name to query entries for |
required |
base_class |
Type[~SUBCLASS_TYPE] |
the base class to look for |
required |
ignore_abstract_classes |
bool |
whether to include abstract classes in the result |
True |
type_id_key |
Optional[str] |
if provided, the found classes will have their id attached as an attribute, using the value of this as the name. if an attribute of this name already exists, it will be used as id without further processing |
None |
type_id_func |
Callable |
a function to take the found class as input, and returns a string representing the id of the class. By default, the module path + "." + class name (snake-case) is used (minus the string 'kiara_modules. |
None |
type_id_no_attach |
bool |
in case you want to use the type_id_key to set the id, but don't want it attached to classes that don't have it, set this to true. In most cases, you won't need this option |
False |
attach_python_metadata |
Union[bool, str] |
whether to attach a PythonClass metadata model to the class. By default, '_python_class' is used as attribute name if this argument is 'True', If this argument is a string, that will be used as name instead. |
False |
Source code in kiara/utils/class_loading.py
def load_all_subclasses_for_entry_point(
entry_point_name: str,
base_class: Type[SUBCLASS_TYPE],
ignore_abstract_classes: bool = True,
type_id_key: Union[str, None] = None,
type_id_func: Callable = None,
type_id_no_attach: bool = False,
attach_python_metadata: Union[bool, str] = False,
) -> Dict[str, Type[SUBCLASS_TYPE]]:
"""Find all subclasses of a base class via package entry points.
Arguments:
entry_point_name: the entry point name to query entries for
base_class: the base class to look for
ignore_abstract_classes: whether to include abstract classes in the result
type_id_key: if provided, the found classes will have their id attached as an attribute, using the value of this as the name. if an attribute of this name already exists, it will be used as id without further processing
type_id_func: a function to take the found class as input, and returns a string representing the id of the class. By default, the module path + "." + class name (snake-case) is used (minus the string 'kiara_modules.<project_name>'', if it exists at the beginning
type_id_no_attach: in case you want to use the type_id_key to set the id, but don't want it attached to classes that don't have it, set this to true. In most cases, you won't need this option
attach_python_metadata: whether to attach a [PythonClass][kiara.models.python_class.PythonClass] metadata model to the class. By default, '_python_class' is used as attribute name if this argument is 'True', If this argument is a string, that will be used as name instead.
"""
log2 = logging.getLogger("stevedore")
out_hdlr = logging.StreamHandler(sys.stdout)
out_hdlr.setFormatter(
logging.Formatter(
f"{entry_point_name} plugin search message/error -> %(message)s"
)
)
out_hdlr.setLevel(logging.INFO)
log2.addHandler(out_hdlr)
if is_debug():
log2.setLevel(logging.DEBUG)
else:
out_hdlr.setLevel(logging.INFO)
log2.setLevel(logging.INFO)
log_message("events.loading.entry_points", entry_point_name=entry_point_name)
mgr = ExtensionManager(
namespace=entry_point_name,
invoke_on_load=False,
propagate_map_exceptions=True,
)
result_entrypoints: Dict[str, Type[SUBCLASS_TYPE]] = {}
result_dynamic: Dict[str, Type[SUBCLASS_TYPE]] = {}
for plugin in mgr:
name = plugin.name
if isinstance(plugin.plugin, type):
# this means an actual (sub-)class was provided in the entrypoint
cls = plugin.plugin
if not issubclass(cls, base_class):
log_message(
"ignore.entrypoint",
entry_point=name,
base_class=base_class,
sub_class=plugin.plugin,
reason=f"Entry point reference not a subclass of '{base_class}'.",
)
continue
_process_subclass(
sub_class=cls,
base_class=base_class,
type_id_key=type_id_key,
type_id_func=type_id_func,
type_id_no_attach=type_id_no_attach,
attach_python_metadata=attach_python_metadata,
ignore_abstract_classes=ignore_abstract_classes,
)
result_entrypoints[name] = cls
elif (
isinstance(plugin.plugin, tuple)
and len(plugin.plugin) >= 1
and callable(plugin.plugin[0])
) or callable(plugin.plugin):
try:
if callable(plugin.plugin):
func = plugin.plugin
args = []
else:
func = plugin.plugin[0]
args = plugin.plugin[1:]
classes = func(*args)
except Exception as e:
log_exception(e)
raise Exception(f"Error trying to load plugin '{plugin.plugin}': {e}")
for sub_class in classes:
type_id = _process_subclass(
sub_class=sub_class,
base_class=base_class,
type_id_key=type_id_key,
type_id_func=type_id_func,
type_id_no_attach=type_id_no_attach,
attach_python_metadata=attach_python_metadata,
ignore_abstract_classes=ignore_abstract_classes,
)
if type_id is None:
continue
if type_id in result_dynamic.keys():
raise Exception(
f"Duplicate type id '{type_id}' for type {entry_point_name}: {result_dynamic[type_id]} -- {sub_class}"
)
result_dynamic[type_id] = sub_class
else:
raise Exception(
f"Can't load subclasses for entry point {entry_point_name} and base class {base_class}: invalid plugin type {type(plugin.plugin)}"
)
for k, v in result_dynamic.items():
if k in result_entrypoints.keys():
msg = f"Duplicate item name '{k}' for type {entry_point_name}: {v} -- {result_entrypoints[k]}."
try:
if type_id_key not in v.__dict__.keys():
msg = f"{msg} Most likely the name is picked up from a subclass, try to add a '{type_id_key}' class attribute to your implementing class, with the name you want to give your type as value."
except Exception:
pass
raise Exception(msg)
result_entrypoints[k] = v
return result_entrypoints
cli
special
¶
F
¶FC
¶Classes¶
OutputFormat (Enum)
¶An enumeration.
Source code in kiara/utils/cli/__init__.py
class OutputFormat(Enum):
@classmethod
def as_dict(cls):
return {i.name: i.value for i in cls}
@classmethod
def keys_as_list(cls):
return cls._member_names_
@classmethod
def values_as_list(cls):
return [i.value for i in cls]
TERMINAL = "terminal"
HTML = "html"
JSON = "json"
JSON_INCL_SCHEMA = "json-incl-schema"
JSON_SCHEMA = "json-schema"
Functions¶
dict_from_cli_args(*args, *, list_keys=None)
¶Source code in kiara/utils/cli/__init__.py
def dict_from_cli_args(
*args: str, list_keys: Union[Iterable[str], None] = None
) -> Dict[str, Any]:
if not args:
return {}
config: Dict[str, Any] = {}
for arg in args:
if "=" in arg:
key, value = arg.split("=", maxsplit=1)
try:
_v = json.loads(value)
except Exception:
_v = value
part_config = {key: _v}
elif os.path.isfile(os.path.realpath(os.path.expanduser(arg))):
path = os.path.realpath(os.path.expanduser(arg))
part_config = get_data_from_file(path)
assert isinstance(part_config, Mapping)
else:
try:
part_config = json.loads(arg)
assert isinstance(part_config, Mapping)
except Exception:
raise Exception(f"Could not parse argument into data: {arg}")
if list_keys is None:
list_keys = []
for k, v in part_config.items():
if k in list_keys:
config.setdefault(k, []).append(v)
else:
if k in config.keys():
logger.warning("duplicate.key", old_value=k, new_value=v)
config[k] = v
return config
is_rich_renderable(item)
¶Source code in kiara/utils/cli/__init__.py
def is_rich_renderable(item: Any):
return isinstance(item, (ConsoleRenderable, RichCast, str))
output_format_option(*param_decls)
¶Attaches an option to the command. All positional arguments are
passed as parameter declarations to :class:Option; all keyword
arguments are forwarded unchanged (except cls).
This is equivalent to creating an :class:Option instance manually
and attaching it to the :attr:Command.params list.
:param cls: the option class to instantiate. This defaults to
:class:Option.
Source code in kiara/utils/cli/__init__.py
def output_format_option(*param_decls: str) -> Callable[[FC], FC]:
"""Attaches an option to the command. All positional arguments are
passed as parameter declarations to :class:`Option`; all keyword
arguments are forwarded unchanged (except ``cls``).
This is equivalent to creating an :class:`Option` instance manually
and attaching it to the :attr:`Command.params` list.
:param cls: the option class to instantiate. This defaults to
:class:`Option`.
"""
if not param_decls:
param_decls = ("--format", "-f")
attrs = {
"help": "The output format. Defaults to 'terminal'.",
"type": click.Choice(OutputFormat.values_as_list()),
}
def decorator(f: FC) -> FC:
# Issue 926, copy attrs, so pre-defined options can re-use the same cls=
option_attrs = attrs.copy()
OptionClass = option_attrs.pop("cls", None) or Option
_param_memo(f, OptionClass(param_decls, **option_attrs)) # type: ignore
return f
return decorator
render_json_schema_str(model)
¶Source code in kiara/utils/cli/__init__.py
def render_json_schema_str(model: BaseModel):
try:
json_str = model.schema_json(option=orjson.OPT_INDENT_2)
except TypeError:
json_str = model.schema_json(indent=2)
return json_str
render_json_str(model)
¶Source code in kiara/utils/cli/__init__.py
def render_json_str(model: BaseModel):
try:
json_str = model.json(option=orjson.OPT_INDENT_2 | orjson.OPT_NON_STR_KEYS)
except TypeError:
json_str = model.json(indent=2)
return json_str
terminal_print(msg=None, in_panel=None, rich_config=None, empty_line_before=False, **config)
¶Source code in kiara/utils/cli/__init__.py
def terminal_print(
msg: Any = None,
in_panel: Union[str, None] = None,
rich_config: Union[Mapping[str, Any], None] = None,
empty_line_before: bool = False,
**config: Any,
) -> None:
if msg is None:
msg = ""
console = get_console()
msg = extract_renderable(msg, render_config=config)
# if hasattr(msg, "create_renderable"):
# msg = msg.create_renderable(**config) # type: ignore
if in_panel is not None:
msg = Panel(msg, title_align="left", title=in_panel)
if empty_line_before:
console.print()
if rich_config:
console.print(msg, **rich_config)
else:
console.print(msg)
terminal_print_model(*models, *, format=None, empty_line_before=None, in_panel=None, **render_config)
¶Source code in kiara/utils/cli/__init__.py
def terminal_print_model(
*models: BaseModel,
format: Union[None, OutputFormat, str] = None,
empty_line_before: Union[bool, None] = None,
in_panel: Union[str, None] = None,
**render_config: Any,
):
if format is None:
format = OutputFormat.TERMINAL
if isinstance(format, str):
format = OutputFormat(format)
if empty_line_before is None:
if format == OutputFormat.TERMINAL:
empty_line_before = True
else:
empty_line_before = False
if format == OutputFormat.TERMINAL:
if len(models) == 1:
terminal_print(
models[0],
in_panel=in_panel,
empty_line_before=empty_line_before,
**render_config,
)
else:
rg = []
if not models:
return
for model in models[0:-1]:
renderable = extract_renderable(model, render_config)
rg.append(renderable)
rg.append(Rule(style="b"))
last = extract_renderable(models[-1], render_config)
rg.append(last)
group = Group(*rg)
terminal_print(group, in_panel=in_panel, **render_config)
elif format == OutputFormat.JSON:
if len(models) == 1:
json_str = render_json_str(models[0])
syntax = Syntax(json_str, "json", background_color="default")
terminal_print(
syntax,
empty_line_before=empty_line_before,
rich_config={"soft_wrap": True},
)
else:
json_strs = []
for model in models:
json_str = render_json_str(model)
json_strs.append(json_str)
json_str_full = "[" + ",\n".join(json_strs) + "]"
syntax = Syntax(json_str_full, "json", background_color="default")
terminal_print(
syntax,
empty_line_before=empty_line_before,
rich_config={"soft_wrap": True},
)
elif format == OutputFormat.JSON_SCHEMA:
if len(models) == 1:
syntax = Syntax(
models[0].schema_json(option=orjson.OPT_INDENT_2),
"json",
background_color="default",
)
terminal_print(
syntax,
empty_line_before=empty_line_before,
rich_config={"soft_wrap": True},
)
else:
json_strs = []
for model in models:
json_strs.append(render_json_schema_str(model))
json_str_full = "[" + ",\n".join(json_strs) + "]"
syntax = Syntax(json_str_full, "json", background_color="default")
terminal_print(
syntax,
empty_line_before=empty_line_before,
rich_config={"soft_wrap": True},
)
elif format == OutputFormat.JSON_INCL_SCHEMA:
if len(models) == 1:
data = models[0].dict()
schema = models[0].schema()
all = {"data": data, "schema": schema}
json_str = orjson_dumps(all, option=orjson.OPT_INDENT_2)
syntax = Syntax(json_str, "json", background_color="default")
terminal_print(
syntax,
empty_line_before=empty_line_before,
rich_config={"soft_wrap": True},
)
else:
all_data = []
for model in models:
data = model.dict()
schema = model.schema()
all_data.append({"data": data, "schema": schema})
json_str = orjson_dumps(all_data, option=orjson.OPT_INDENT_2)
# print(json_str)
syntax = Syntax(json_str, "json", background_color="default")
terminal_print(
syntax,
empty_line_before=empty_line_before,
rich_config={"soft_wrap": True},
)
elif format == OutputFormat.HTML:
all_html = ""
for model in models:
if hasattr(model, "create_html"):
html = model.create_html() # type: ignore
all_html = f"{all_html}\n{html}"
else:
raise NotImplementedError()
syntax = Syntax(all_html, "html", background_color="default")
terminal_print(
syntax, empty_line_before=empty_line_before, rich_config={"soft_wrap": True}
)
Modules¶
rich_click
¶rich_format_operation_help(obj, ctx, operation, cmd_help)
¶Print nicely formatted help text using rich.
Based on original code from rich-cli, by @willmcgugan. https://github.com/Textualize/rich-cli/blob/8a2767c7a340715fc6fbf4930ace717b9b2fc5e5/src/rich_cli/main.py#L162-L236
Replacement for the click function format_help(). Takes a command or group and builds the help text output.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
obj |
click.Command or click.Group |
Command or group to build help text for |
required |
ctx |
click.Context |
Click Context object |
required |
table |
a rich table, including all the inputs of the current operation |
required |
Source code in kiara/utils/cli/rich_click.py
def rich_format_operation_help(
obj: Union[click.Command, click.Group],
ctx: click.Context,
operation: KiaraOperation,
cmd_help: str,
) -> None:
"""Print nicely formatted help text using rich.
Based on original code from rich-cli, by @willmcgugan.
https://github.com/Textualize/rich-cli/blob/8a2767c7a340715fc6fbf4930ace717b9b2fc5e5/src/rich_cli/__main__.py#L162-L236
Replacement for the click function format_help().
Takes a command or group and builds the help text output.
Args:
obj (click.Command or click.Group): Command or group to build help text for
ctx (click.Context): Click Context object
table: a rich table, including all the inputs of the current operation
"""
renderables: List[RenderableType] = []
# Header text if we have it
if HEADER_TEXT:
renderables.append(
Padding(_make_rich_rext(HEADER_TEXT, STYLE_HEADER_TEXT), (1, 1, 0, 1))
)
# Print usage
_cmd = cmd_help
renderables.append(Padding(_cmd, 1))
# renderables.append(obj.get_usage(ctx))
# renderables.append(Panel(Padding(highlighter(obj.get_usage(ctx)), 1), style=STYLE_USAGE_COMMAND, box=box.MINIMAL))
# Print command / group help if we have some
desc = operation.operation.doc.full_doc
renderables.append(
Padding(
Align(Markdown(desc), width=MAX_WIDTH, pad=False),
(0, 1, 1, 1),
)
)
# if obj.help:
#
# # Print with a max width and some padding
# renderables.append(
# Padding(
# Align(_get_help_text(obj), width=MAX_WIDTH, pad=False),
# (0, 1, 1, 1),
# )
# )
# Look through OPTION_GROUPS for this command
# stick anything unmatched into a default group at the end
option_groups = OPTION_GROUPS.get(ctx.command_path, []).copy()
option_groups.append({"options": []})
argument_group_options = []
for param in obj.get_params(ctx):
# Skip positional arguments - they don't have opts or helptext and are covered in usage
# See https://click.palletsprojects.com/en/8.0.x/documentation/#documenting-arguments
if type(param) is click.core.Argument and not SHOW_ARGUMENTS:
continue
# Skip if option is hidden
if getattr(param, "hidden", False):
continue
# Already mentioned in a config option group
for option_group in option_groups:
if any([opt in option_group.get("options", []) for opt in param.opts]):
break
# No break, no mention - add to the default group
else:
if type(param) is click.core.Argument and not GROUP_ARGUMENTS_OPTIONS:
argument_group_options.append(param.opts[0])
else:
list_of_option_groups: List = option_groups[-1]["options"] # type: ignore
list_of_option_groups.append(param.opts[0])
# If we're not grouping arguments and we got some, prepend before default options
if len(argument_group_options) > 0:
extra_option_group = {
"name": ARGUMENTS_PANEL_TITLE,
"options": argument_group_options,
}
option_groups.insert(len(option_groups) - 1, extra_option_group) # type: ignore
# Print each option group panel
for option_group in option_groups:
options_rows = []
for opt in option_group.get("options", []):
# Get the param
for param in obj.get_params(ctx):
if any([opt in param.opts]):
break
# Skip if option is not listed in this group
else:
continue
# Short and long form
opt_long_strs = []
opt_short_strs = []
for idx, opt in enumerate(param.opts):
opt_str = opt
try:
opt_str += "/" + param.secondary_opts[idx]
except IndexError:
pass
if "--" in opt:
opt_long_strs.append(opt_str)
else:
opt_short_strs.append(opt_str)
# Column for a metavar, if we have one
metavar = Text(style=STYLE_METAVAR, overflow="fold")
metavar_str = param.make_metavar()
# Do it ourselves if this is a positional argument
if type(param) is click.core.Argument and metavar_str == param.name.upper(): # type: ignore
metavar_str = param.type.name.upper()
# Skip booleans and choices (handled above)
if metavar_str != "BOOLEAN":
metavar.append(metavar_str)
# Range - from
# https://github.com/pallets/click/blob/c63c70dabd3f86ca68678b4f00951f78f52d0270/src/click/core.py#L2698-L2706 # noqa: E501
try:
# skip count with default range type
if isinstance(param.type, click.types._NumberRangeBase) and not (
param.count and param.type.min == 0 and param.type.max is None # type: ignore
):
range_str = param.type._describe_range()
if range_str:
metavar.append(RANGE_STRING.format(range_str))
except AttributeError:
# click.types._NumberRangeBase is only in Click 8x onwards
pass
# Required asterisk
required: RenderableType = ""
if param.required:
required = Text(REQUIRED_SHORT_STRING, style=STYLE_REQUIRED_SHORT)
# Highlighter to make [ | ] and <> dim
class MetavarHighlighter(RegexHighlighter):
highlights = [
r"^(?P<metavar_sep>(\[|<))",
r"(?P<metavar_sep>\|)",
r"(?P<metavar_sep>(\]|>)$)",
]
metavar_highlighter = MetavarHighlighter()
rows = [
required,
highlighter(highlighter(",".join(opt_long_strs))),
highlighter(highlighter(",".join(opt_short_strs))),
metavar_highlighter(metavar),
_get_parameter_help(param, ctx), # type: ignore
]
# Remove metavar if specified in config
if not SHOW_METAVARS_COLUMN:
rows.pop(3)
options_rows.append(rows)
if len(options_rows) > 0:
t_styles = {
"show_lines": STYLE_OPTIONS_TABLE_SHOW_LINES,
"leading": STYLE_OPTIONS_TABLE_LEADING,
"box": STYLE_OPTIONS_TABLE_BOX,
"border_style": STYLE_OPTIONS_TABLE_BORDER_STYLE,
"row_styles": STYLE_OPTIONS_TABLE_ROW_STYLES,
"pad_edge": STYLE_OPTIONS_TABLE_PAD_EDGE,
"padding": STYLE_OPTIONS_TABLE_PADDING,
}
t_styles.update(option_group.get("table_styles", {})) # type: ignore
box_style = getattr(box, t_styles.pop("box"), None) # type: ignore
options_table = Table(
highlight=True,
show_header=False,
expand=True,
box=box_style,
**t_styles, # type: ignore
)
# Strip the required column if none are required
if all([x[0] == "" for x in options_rows]):
options_rows = [x[1:] for x in options_rows]
for row in options_rows:
options_table.add_row(*row)
renderables.append(
Panel(
options_table,
border_style=STYLE_OPTIONS_PANEL_BORDER, # type: ignore
title=option_group.get("name", OPTIONS_PANEL_TITLE), # type: ignore
title_align=ALIGN_OPTIONS_PANEL, # type: ignore
width=MAX_WIDTH, # type: ignore
)
)
#
# Groups only:
# List click command groups
#
if hasattr(obj, "list_commands"):
# Look through COMMAND_GROUPS for this command
# stick anything unmatched into a default group at the end
cmd_groups = COMMAND_GROUPS.get(ctx.command_path, []).copy()
cmd_groups.append({"commands": []})
for command in obj.list_commands(ctx): # type: ignore
for cmd_group in cmd_groups:
if command in cmd_group.get("commands", []):
break
else:
commands: List = cmd_groups[-1]["commands"] # type: ignore
commands.append(command)
# Print each command group panel
for cmd_group in cmd_groups:
t_styles = {
"show_lines": STYLE_COMMANDS_TABLE_SHOW_LINES,
"leading": STYLE_COMMANDS_TABLE_LEADING,
"box": STYLE_COMMANDS_TABLE_BOX,
"border_style": STYLE_COMMANDS_TABLE_BORDER_STYLE,
"row_styles": STYLE_COMMANDS_TABLE_ROW_STYLES,
"pad_edge": STYLE_COMMANDS_TABLE_PAD_EDGE,
"padding": STYLE_COMMANDS_TABLE_PADDING,
}
t_styles.update(cmd_group.get("table_styles", {})) # type: ignore
box_style = getattr(box, t_styles.pop("box"), None) # type: ignore
commands_table = Table(
highlight=False,
show_header=False,
expand=True,
box=box_style, # type: ignore
**t_styles, # type: ignore
)
# Define formatting in first column, as commands don't match highlighter regex
commands_table.add_column(style="bold cyan", no_wrap=True)
for command in cmd_group.get("commands", []):
# Skip if command does not exist
if command not in obj.list_commands(ctx): # type: ignore
continue
cmd = obj.get_command(ctx, command) # type: ignore
assert cmd is not None
if cmd.hidden:
continue
# Use the truncated short text as with vanilla text if requested
if USE_CLICK_SHORT_HELP:
helptext = cmd.get_short_help_str()
else:
# Use short_help function argument if used, or the full help
helptext = cmd.short_help or cmd.help or ""
commands_table.add_row(command, _make_command_help(helptext))
if commands_table.row_count > 0:
renderables.append(
Panel(
commands_table,
border_style=STYLE_COMMANDS_PANEL_BORDER, # type: ignore
title=cmd_group.get("name", COMMANDS_PANEL_TITLE), # type: ignore
title_align=ALIGN_COMMANDS_PANEL, # type: ignore
width=MAX_WIDTH, # type: ignore
)
)
inputs_table = operation.create_renderable(
show_operation_name=False,
show_operation_doc=False,
show_inputs=True,
show_outputs_schema=False,
show_headers=False,
)
inputs_panel = Panel(
inputs_table,
title="Inputs",
border_style=STYLE_COMMANDS_PANEL_BORDER, # type: ignore
title_align=ALIGN_COMMANDS_PANEL, # type: ignore
width=MAX_WIDTH, # type: ignore
)
renderables.append(inputs_panel)
# Epilogue if we have it
if obj.epilog:
# Remove single linebreaks, replace double with single
lines = obj.epilog.split("\n\n")
epilogue = "\n".join([x.replace("\n", " ").strip() for x in lines])
renderables.append(
Padding(Align(highlighter(epilogue), width=MAX_WIDTH, pad=False), 1)
)
# Footer text if we have it
if FOOTER_TEXT:
renderables.append(
Padding(_make_rich_rext(FOOTER_TEXT, STYLE_FOOTER_TEXT), (1, 1, 0, 1))
)
group = Group(*renderables)
terminal_print(group)
run
¶calculate_aliases(kiara_op, alias_tokens)
¶Source code in kiara/utils/cli/run.py
def calculate_aliases(
kiara_op: KiaraOperation, alias_tokens: Iterable[str]
) -> Mapping[str, List[str]]:
if not alias_tokens:
aliases: Dict[str, List[str]] = {}
full_aliases: List[str] = []
else:
aliases = {}
full_aliases = []
for a in alias_tokens:
if "=" not in a:
full_aliases.append(a)
else:
tokens = a.split("=")
if len(tokens) != 2:
print()
print(f"Invalid alias format, can only contain a single '=': {a}")
sys.exit(1)
aliases.setdefault(tokens[0], []).append(tokens[1])
# =========================================================================
# check save user input
final_aliases = {}
if alias_tokens:
op_output_names = kiara_op.operation.outputs_schema.keys()
invalid_fields = []
for field_name, alias in aliases.items():
if field_name not in op_output_names:
invalid_fields.append(field_name)
else:
final_aliases[field_name] = alias
for _alias in full_aliases:
for field_name in op_output_names:
final_aliases.setdefault(field_name, []).append(
f"{_alias}.{field_name}"
)
if invalid_fields:
print()
print(
f"Can't run workflow, invalid field name(s) when specifying aliases: {', '.join(invalid_fields)}. Valid field names: {', '.join(op_output_names)}"
)
sys.exit(1)
return final_aliases
execute_job(kiara_op, silent, save_results, aliases)
¶Execute the job
Source code in kiara/utils/cli/run.py
def execute_job(
kiara_op: KiaraOperation,
silent: bool,
save_results: bool,
aliases: Union[None, Mapping[str, List[str]]],
) -> ValueMap:
"""Execute the job"""
job_id = kiara_op.queue_job()
try:
outputs = kiara_op.retrieve_result(job_id=job_id)
except FailedJobException as fje:
print()
terminal_print(fje, in_panel="Processing error")
sys.exit(1)
except Exception as e:
print()
terminal_print(e)
sys.exit(1)
if not silent:
if len(outputs) > 1:
title = "[b]Results[/b]"
else:
title = "[b]Result[/b]"
# for field_name, value in outputs.items():
# results.append("")
# results.append(f"* [b i]{field_name}[/b i]")
# results.append(kiara_obj.data_registry.render_data(value.value_id))
terminal_print(
outputs, in_panel=title, empty_line_before=True, show_data_type=True
)
# for k, v in outputs.items():
# rendered = kiara_obj.data_registry.render_data(v)
# rich_print(rendered)
if save_results:
try:
saved_results = kiara_op.save_result(job_id=job_id, aliases=aliases)
if len(saved_results) == 1:
title = "[b]Stored result value[/b]"
else:
title = "[b]Stored result values[/b]"
terminal_print(saved_results, in_panel=title, empty_line_before=True)
except Exception as e:
log_exception(e)
terminal_print(f"[red]Error saving results[/red]: {e}")
sys.exit(1)
return outputs
set_and_validate_inputs(kiara_op, inputs, explain, print_help, click_context, cmd_help)
¶Source code in kiara/utils/cli/run.py
def set_and_validate_inputs(
kiara_op: KiaraOperation,
inputs: Iterable[str],
explain: bool,
print_help: bool,
click_context: ClickContext,
cmd_help: str,
):
# =========================================================================
# prepare inputs
list_keys = []
for (
name,
value_schema,
) in kiara_op.operation.operation_details.inputs_schema.items():
if value_schema.type in ["list"]:
list_keys.append(name)
inputs_dict = dict_from_cli_args(*inputs, list_keys=list_keys)
kiara_op.set_inputs(**inputs_dict)
if print_help:
rich_format_operation_help(
obj=click_context.command,
ctx=click_context,
operation=kiara_op,
cmd_help=cmd_help,
)
sys.exit(0)
if explain:
terminal_print()
rg = Group(
"",
kiara_op.create_renderable(
show_operation_name=True, show_inputs=True, show_outputs_schema=True
),
)
terminal_print(rg, in_panel=f"Operation info: [b]{kiara_op.operation_name}[/b]")
sys.exit(0)
try:
operation_inputs = kiara_op.operation_inputs
except InvalidValuesException as ive:
terminal_print()
rg = Group(
"",
f"Can't run operation: {ive}",
"",
Rule(),
"",
kiara_op.create_renderable(
show_operation_name=True, show_inputs=True, show_outputs_schema=True
),
)
terminal_print(rg, in_panel=f"Run info: [b]{kiara_op.operation_name}[/b]")
sys.exit(1)
invalid = operation_inputs.check_invalid()
if invalid:
terminal_print()
rg = Group(
"",
"Can't run operation, invalid or insufficient inputs.",
"",
Rule(),
"",
kiara_op.create_renderable(
show_operation_name=True, show_inputs=True, show_outputs_schema=True
),
)
terminal_print(rg, in_panel=f"Run info: [b]{kiara_op.operation_name}[/b]")
sys.exit(1)
validate_operation_in_terminal(kiara, module_or_operation, module_config)
¶Source code in kiara/utils/cli/run.py
def validate_operation_in_terminal(
kiara: Kiara, module_or_operation: str, module_config: Mapping[str, Any]
) -> KiaraOperation:
kiara_op = KiaraOperation(
kiara=kiara,
operation_name=module_or_operation,
operation_config=module_config,
)
try:
# validate that operation config is valid, ignoring inputs for now
kiara_op.operation # noqa
except NoSuchExecutionTargetException as nset:
print()
terminal_print(nset)
print()
print("Existing operations:")
print()
for n in nset.avaliable_targets:
terminal_print(f" - [i]{n}[/i]")
sys.exit(1)
except ValidationError as ve:
renderables = [""]
renderables.append("Invalid module configuration:")
renderables.append("")
for error in ve.errors():
loc = ", ".join(error["loc"]) # type: ignore
renderables.append(f" [b]{loc}[/b]: [red]{error['msg']}[/red]")
try:
m = kiara.module_registry.get_module_class(kiara_op.operation_name)
schema = create_table_from_base_model_cls(m._config_cls)
renderables.append("")
renderables.append(f"Module configuration schema for '[b i]{m._module_type_name}[/b i]':") # type: ignore
renderables.append("")
renderables.append(schema)
except Exception:
pass
msg = Group(*renderables)
terminal_print()
terminal_print(msg, in_panel="[b red]Module configuration error[/b red]")
sys.exit(1)
except Exception as e:
log_exception(e)
terminal_print()
terminal_print(
f"Error when trying to validate the operation [i]'{kiara_op.operation_name}'[/i]:\n"
)
terminal_print(f" [red]{e}[/red]")
sys.exit(1)
return kiara_op
concurrency
¶
Classes¶
ThreadSaveCounter
¶A thread-safe counter, can be used in kiara modules to update completion percentage.
Source code in kiara/utils/concurrency.py
class ThreadSaveCounter(object):
"""A thread-safe counter, can be used in kiara modules to update completion percentage."""
def __init__(self):
self._current = 0
self._lock = threading.Lock()
@property
def current(self):
return self._current
def current_percent(self, total: int) -> int:
return int((self.current / total) * 100)
def increment(self):
with self._lock:
self._current += 1
return self._current
def decrement(self):
with self._lock:
self._current -= 1
return self._current
current
property
readonly
¶current_percent(self, total)
¶Source code in kiara/utils/concurrency.py
def current_percent(self, total: int) -> int:
return int((self.current / total) * 100)
decrement(self)
¶Source code in kiara/utils/concurrency.py
def decrement(self):
with self._lock:
self._current -= 1
return self._current
increment(self)
¶Source code in kiara/utils/concurrency.py
def increment(self):
with self._lock:
self._current += 1
return self._current
data
¶
logger
¶pretty_print_data(kiara, value_id, target_type='terminal_renderable', **render_config)
¶Source code in kiara/utils/data.py
def pretty_print_data(
kiara: "Kiara",
value_id: uuid.UUID,
target_type="terminal_renderable",
**render_config: Any,
) -> Any:
value = kiara.data_registry.get_value(value=value_id)
op_type: PrettyPrintOperationType = kiara.operation_registry.get_operation_type("pretty_print") # type: ignore
try:
op: Union[Operation, None] = op_type.get_operation_for_render_combination(
source_type=value.value_schema.type, target_type=target_type
)
except Exception as e:
logger.debug(
"error.pretty_print",
source_type=value.value_schema.type,
target_type=target_type,
error=e,
)
op = None
if target_type == "terminal_renderable":
try:
op = op_type.get_operation_for_render_combination(
source_type="any", target_type="string"
)
except Exception:
pass
if op is None:
raise Exception(
f"Can't find operation to render '{value.value_schema.type}' as '{target_type}."
)
result = op.run(kiara=kiara, inputs={"value": value})
rendered = result.get_value_data("rendered_value")
return rendered
db
¶
get_kiara_db_url(base_path)
¶Source code in kiara/utils/db.py
def get_kiara_db_url(base_path: str):
abs_path = os.path.abspath(os.path.expanduser(base_path))
db_url = f"sqlite+pysqlite:///{abs_path}/kiara.db"
return db_url
orm_json_deserialize(obj)
¶Source code in kiara/utils/db.py
def orm_json_deserialize(obj: str) -> Any:
return orjson.loads(obj)
orm_json_serialize(obj)
¶Source code in kiara/utils/db.py
def orm_json_serialize(obj: Any) -> str:
if hasattr(obj, "json"):
return obj.json()
if isinstance(obj, str):
return obj
elif isinstance(obj, Mapping):
return orjson_dumps(obj, default=None)
else:
raise Exception(f"Unsupported type for json serialization: {type(obj)}")
debug
¶
DEFAULT_VALUE_MAP_RENDER_CONFIG
¶create_module_preparation_table(kiara, job_config, job_id, module, **render_config)
¶Source code in kiara/utils/debug.py
def create_module_preparation_table(
kiara: "Kiara",
job_config: JobConfig,
job_id: uuid.UUID,
module: "KiaraModule",
**render_config: Any
) -> Table:
dev_config = get_dev_config()
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
table.add_row("job_id", str(job_id))
module_details = dev_config.log.pre_run.module_info
if module_details not in [DetailLevel.NONE.value, DetailLevel.NONE]:
if module_details in [DetailLevel.MINIMAL.value, DetailLevel.MINIMAL]:
table.add_row("module", job_config.module_type)
doc = module.operation.doc
table.add_row(
"module desc",
doc.description
# kiara.context_info.module_types.item_infos[
# job_config.module_type
# ].documentation.description,
)
elif module_details in [DetailLevel.FULL.value, DetailLevel.FULL]:
table.add_row("module", job_config.module_type)
doc = module.operation.doc
table.add_row(
"module doc",
doc.full_doc
# kiara.context_info.module_types.item_infos[
# job_config.module_type
# ].documentation.full_doc,
)
if module_config_is_empty(job_config.module_config):
table.add_row("module_config", "-- no config --")
else:
module = kiara.module_registry.create_module(manifest=job_config)
table.add_row("module_config", module.config)
inputs_details = dev_config.log.pre_run.inputs_info
if inputs_details not in [DetailLevel.NONE.value, DetailLevel.NONE]:
if inputs_details in [DetailLevel.MINIMAL, DetailLevel.MINIMAL.value]:
render_config["show_type"] = False
value_map_rend = create_value_map_renderable(
value_map=job_config.inputs, **render_config
)
table.add_row("inputs", value_map_rend)
elif inputs_details in [DetailLevel.FULL, DetailLevel.FULL.value]:
value_map = kiara.data_registry.load_values(values=job_config.inputs)
table.add_row("inputs", value_map.create_renderable(**render_config))
return table
create_post_run_table(kiara, job, module, job_config, **render_config)
¶Source code in kiara/utils/debug.py
def create_post_run_table(
kiara: "Kiara",
job: ActiveJob,
module: "KiaraModule",
job_config: JobConfig,
**render_config: Any
) -> Table:
dev_config = get_dev_config()
table = Table(show_header=False, box=box.SIMPLE)
table.add_column("key", style="i")
table.add_column("value")
table.add_row("job_id", str(job.job_id))
module_details = dev_config.log.post_run.module_info
if module_details not in [DetailLevel.NONE.value, DetailLevel.NONE]:
if module_details in [DetailLevel.MINIMAL.value, DetailLevel.MINIMAL]:
table.add_row("module", module.module_type_name)
table.add_row(
"module desc",
kiara.context_info.module_types.item_infos[
module.module_type_name
].documentation.description,
)
elif module_details in [DetailLevel.FULL.value, DetailLevel.FULL]:
table.add_row("module", module.module_type_name)
table.add_row(
"module doc",
kiara.context_info.module_types.item_infos[
module.module_type_name
].documentation.full_doc,
)
if module_config_is_empty(module.config.dict()):
table.add_row("module_config", "-- no config --")
else:
table.add_row("module_config", module.config)
inputs_details = dev_config.log.post_run.inputs_info
if inputs_details not in [DetailLevel.NONE.value, DetailLevel.NONE]:
if inputs_details in [DetailLevel.MINIMAL, DetailLevel.MINIMAL.value]:
render_config["show_type"] = False
value_map_rend: RenderableType = create_value_map_renderable(
value_map=job_config.inputs, **render_config
)
table.add_row("inputs", value_map_rend)
elif inputs_details in [DetailLevel.FULL, DetailLevel.FULL.value]:
value_map = kiara.data_registry.load_values(values=job_config.inputs)
table.add_row("inputs", value_map.create_renderable(**render_config))
outputs_details = dev_config.log.post_run.outputs_info
if outputs_details not in [DetailLevel.NONE.value, DetailLevel.NONE]:
if outputs_details in [DetailLevel.MINIMAL, DetailLevel.MINIMAL.value]:
render_config["show_type"] = False
if job.results is None:
value_map_rend = "-- no results --"
else:
value_map_rend = create_value_map_renderable(
value_map=job.results, **render_config
)
table.add_row("outputs", value_map_rend)
elif outputs_details in [DetailLevel.FULL, DetailLevel.FULL.value]:
if job.results is None:
value_map_rend = "-- no results --"
else:
value_map = kiara.data_registry.load_values(values=job.results)
value_map_rend = value_map.create_renderable(**render_config)
table.add_row("outputs", value_map_rend)
return table
create_value_map_renderable(value_map, **render_config)
¶Source code in kiara/utils/debug.py
def create_value_map_renderable(value_map: Mapping[str, Any], **render_config: Any):
show_type = render_config.get("show_type", True)
rc = dict(DEFAULT_VALUE_MAP_RENDER_CONFIG)
rc.update(render_config)
table = Table(show_header=True, box=box.SIMPLE)
table.add_column("field name", style="i")
if show_type:
table.add_column("type")
table.add_column("value")
for k, v in value_map.items():
row: List[Any] = [k]
if isinstance(v, Value):
if show_type:
row.append("value object")
row.append(v.create_renderable(**rc))
elif isinstance(v, uuid.UUID):
if show_type:
row.append("value id")
row.append(str(v))
else:
if show_type:
row.append("raw data")
row.append(str(v))
table.add_row(*row)
return table
terminal_print_manifest(manifest)
¶Source code in kiara/utils/debug.py
def terminal_print_manifest(manifest: Manifest):
terminal_print(manifest.create_renderable())
develop
special
¶
KIARA_DEV_SETTINGS
¶Classes¶
DetailLevel (Enum)
¶
KiaraDevLogSettings (BaseModel)
pydantic-model
¶Source code in kiara/utils/develop/__init__.py
class KiaraDevLogSettings(BaseModel):
PROFILES: ClassVar[Dict[str, Any]] = {
"full": {
"log_pre_run": True,
"pre_run": {
"pipeline_steps": True,
"module_info": "full",
"inputs_info": "full",
},
"log_post_run": True,
"post_run": {
"pipeline_steps": True,
"module_info": "minimal",
"inputs_info": "minimal",
"outputs_info": "full",
},
},
"internal": {
"pre_run": {"internal_modules": True},
"post_run": {"internal_modules": True},
},
}
class Config:
extra = Extra.forbid
validate_assignment = True
use_enum_values = True
exc: DetailLevel = Field(
description="How detailed to print exceptions", default=DetailLevel.MINIMAL
)
log_pre_run: bool = Field(
description="Print details about a module and its inputs before running it.",
default=True,
)
pre_run: PreRunMsgDetails = Field(
description="Fine-grained settings about what to display in the pre-run message.",
default_factory=PreRunMsgDetails,
)
log_post_run: bool = Field(
description="Print details about the results of a module run.", default=True
)
post_run: PostRunMsgDetails = Field(
description="Fine-grained settings aobut what to display in the post-run message.",
default_factory=PostRunMsgDetails,
)
PROFILES: ClassVar[Dict[str, Any]]
¶exc: DetailLevel
pydantic-field
¶How detailed to print exceptions
log_post_run: bool
pydantic-field
¶Print details about the results of a module run.
log_pre_run: bool
pydantic-field
¶Print details about a module and its inputs before running it.
post_run: PostRunMsgDetails
pydantic-field
¶Fine-grained settings aobut what to display in the post-run message.
pre_run: PreRunMsgDetails
pydantic-field
¶Fine-grained settings about what to display in the pre-run message.
KiaraDevSettings (BaseSettings)
pydantic-model
¶Source code in kiara/utils/develop/__init__.py
class KiaraDevSettings(BaseSettings):
class Config:
extra = Extra.forbid
validate_assignment = True
env_prefix = "dev_"
use_enum_values = True
env_nested_delimiter = "__"
@classmethod
def customise_sources(
cls,
init_settings,
env_settings,
file_secret_settings,
):
return (
init_settings,
profile_settings_source,
dev_config_file_settings_source,
env_settings,
)
log: KiaraDevLogSettings = Field(
description="Settings about what messages to print in 'develop' mode, and what details to include.",
default_factory=KiaraDevLogSettings,
)
job_cache: bool = Field(
description="Whether to always disable the job cache (ignores the runtime_job_cache setting in the kiara configuration).",
default=True,
)
def create_renderable(self, **render_config: Any):
from kiara.utils.output import create_recursive_table_from_model_object
return create_recursive_table_from_model_object(
self, render_config=render_config
)
job_cache: bool
pydantic-field
¶Whether to always disable the job cache (ignores the runtime_job_cache setting in the kiara configuration).
log: KiaraDevLogSettings
pydantic-field
¶Settings about what messages to print in 'develop' mode, and what details to include.
Config
¶Source code in kiara/utils/develop/__init__.py
class Config:
extra = Extra.forbid
validate_assignment = True
env_prefix = "dev_"
use_enum_values = True
env_nested_delimiter = "__"
@classmethod
def customise_sources(
cls,
init_settings,
env_settings,
file_secret_settings,
):
return (
init_settings,
profile_settings_source,
dev_config_file_settings_source,
env_settings,
)
env_nested_delimiter
¶env_prefix
¶extra
¶use_enum_values
¶validate_assignment
¶customise_sources(init_settings, env_settings, file_secret_settings)
classmethod
¶Source code in kiara/utils/develop/__init__.py
@classmethod
def customise_sources(
cls,
init_settings,
env_settings,
file_secret_settings,
):
return (
init_settings,
profile_settings_source,
dev_config_file_settings_source,
env_settings,
)
create_renderable(self, **render_config)
¶Source code in kiara/utils/develop/__init__.py
def create_renderable(self, **render_config: Any):
from kiara.utils.output import create_recursive_table_from_model_object
return create_recursive_table_from_model_object(
self, render_config=render_config
)
PostRunMsgDetails (BaseModel)
pydantic-model
¶Source code in kiara/utils/develop/__init__.py
class PostRunMsgDetails(BaseModel):
class Config:
extra = Extra.forbid
validate_assignment = True
use_enum_values = True
pipeline_steps: bool = Field(
description="Whether to also display information for modules that are run as part of a pipeline",
default=False,
)
module_info: DetailLevel = Field(
description="Whether to display details about the module that was run.",
default=DetailLevel.NONE,
)
internal_modules: bool = Field(
description="Whether to also print details about runs of internal module.",
default=False,
)
inputs_info: DetailLevel = Field(
description="Whether to display details about the run inputs.",
default=DetailLevel.NONE,
)
outputs_info: DetailLevel = Field(
description="Whether to display details about the run outputs.",
default=DetailLevel.MINIMAL,
)
inputs_info: DetailLevel
pydantic-field
¶Whether to display details about the run inputs.
internal_modules: bool
pydantic-field
¶Whether to also print details about runs of internal module.
module_info: DetailLevel
pydantic-field
¶Whether to display details about the module that was run.
outputs_info: DetailLevel
pydantic-field
¶Whether to display details about the run outputs.
pipeline_steps: bool
pydantic-field
¶Whether to also display information for modules that are run as part of a pipeline
PreRunMsgDetails (BaseModel)
pydantic-model
¶Source code in kiara/utils/develop/__init__.py
class PreRunMsgDetails(BaseModel):
class Config:
extra = Extra.forbid
validate_assignment = True
use_enum_values = True
pipeline_steps: bool = Field(
description="Whether to also display information for modules that are run as part of a pipeline.",
default=False,
)
module_info: DetailLevel = Field(
description="Whether to display details about the module to be run.",
default=DetailLevel.MINIMAL,
)
internal_modules: bool = Field(
description="Whether to also print details about runs of internal modules.",
default=False,
)
inputs_info: DetailLevel = Field(
description="Whether to display details about the run inputs.",
default=DetailLevel.MINIMAL,
)
inputs_info: DetailLevel
pydantic-field
¶Whether to display details about the run inputs.
internal_modules: bool
pydantic-field
¶Whether to also print details about runs of internal modules.
module_info: DetailLevel
pydantic-field
¶Whether to display details about the module to be run.
pipeline_steps: bool
pydantic-field
¶Whether to also display information for modules that are run as part of a pipeline.
Functions¶
dev_config_file_settings_source(settings)
¶A simple settings source that loads variables from a JSON file at the project's root.
Here we happen to choose to use the env_file_encoding from Config
when reading config.json
Source code in kiara/utils/develop/__init__.py
def dev_config_file_settings_source(settings: BaseSettings) -> Dict[str, Any]:
"""
A simple settings source that loads variables from a JSON file
at the project's root.
Here we happen to choose to use the `env_file_encoding` from Config
when reading `config.json`
"""
if os.path.exists(KIARA_DEV_CONFIG_FILE):
dev_config = get_data_from_file(KIARA_DEV_CONFIG_FILE)
else:
dev_config = {}
return dev_config
log_dev_message(msg, title=None)
¶Source code in kiara/utils/develop/__init__.py
def log_dev_message(msg: RenderableType, title: Union[str, None] = None):
if not is_develop():
return
if not title:
title = "Develop-mode message"
panel = Panel(Group("", msg), title=f"[yellow]{title}[/yellow]", title_align="left")
from kiara.utils.cli import terminal_print
terminal_print(panel)
profile_settings_source(settings)
¶Source code in kiara/utils/develop/__init__.py
def profile_settings_source(settings: BaseSettings) -> Dict[str, Any]:
profile_name = os.environ.get("DEVELOP", None)
if not profile_name:
profile_name = os.environ.get("develop", None)
if not profile_name:
profile_name = os.environ.get("DEV", None)
if not profile_name:
profile_name = os.environ.get("dev", None)
if not profile_name:
profile_name = os.environ.get("DEV_PROFILE", None)
if not profile_name:
profile_name = os.environ.get("dev_profile", None)
result: Dict[str, Any] = {}
if not profile_name:
return result
profile_name = profile_name.lower()
from pydantic.fields import ModelField
model: ModelField
for model in KiaraDevSettings.__fields__.values():
if not issubclass(model.type_, BaseModel):
continue
profiles = getattr(model.type_, "PROFILES", None)
if not profiles:
continue
p = profiles.get(profile_name, None)
if not p:
continue
result[model.name] = p
return result
dicts
¶
merge_dicts(*dicts)
¶Source code in kiara/utils/dicts.py
def merge_dicts(*dicts: Mapping[str, Any]) -> Dict[str, Any]:
if not dicts:
return {}
current: Dict[str, Any] = {}
for d in dicts:
dpath.util.merge(current, copy.deepcopy(d))
return current
doc
¶
extract_doc_from_cls(cls, only_first_line=False)
¶Source code in kiara/utils/doc.py
def extract_doc_from_cls(cls: typing.Type, only_first_line: bool = False):
doc = cls.__doc__
if not doc:
doc = DEFAULT_NO_DESC_VALUE
else:
doc = cleandoc(doc)
if only_first_line:
return first_line(doc)
else:
return doc.strip()
files
¶
yaml
¶get_data_from_file(path, content_type=None)
¶Source code in kiara/utils/files.py
def get_data_from_file(
path: Union[str, Path], content_type: Union[str, None] = None
) -> Any:
if isinstance(path, str):
path = Path(os.path.expanduser(path))
content = path.read_text()
if content_type:
assert content_type in ["json", "yaml"]
else:
if path.name.endswith(".json"):
content_type = "json"
elif path.name.endswith(".yaml") or path.name.endswith(".yml"):
content_type = "yaml"
else:
raise ValueError(
"Invalid data format, only 'json' or 'yaml' are supported currently."
)
if content_type == "json":
data = json.loads(content)
else:
data = yaml.load(content)
return data
global_metadata
¶
get_metadata_for_python_module_or_class(module_or_class)
¶Source code in kiara/utils/global_metadata.py
@lru_cache()
def get_metadata_for_python_module_or_class(
module_or_class: typing.Union[ModuleType, typing.Type]
) -> typing.List[typing.Dict[str, typing.Any]]:
metadata: typing.List[typing.Dict[str, typing.Any]] = []
if isinstance(module_or_class, type):
if hasattr(module_or_class, KIARA_MODULE_METADATA_ATTRIBUTE):
md = getattr(module_or_class, KIARA_MODULE_METADATA_ATTRIBUTE)
assert isinstance(md, typing.Mapping)
metadata.append(md) # type: ignore
_module_or_class: typing.Union[
str, ModuleType, typing.Type
] = module_or_class.__module__
else:
_module_or_class = module_or_class
current_module = _module_or_class
while current_module:
if isinstance(current_module, str):
current_module = importlib.import_module(current_module)
if hasattr(current_module, KIARA_MODULE_METADATA_ATTRIBUTE):
md = getattr(current_module, KIARA_MODULE_METADATA_ATTRIBUTE)
assert isinstance(md, typing.Mapping)
metadata.append(md) # type: ignore
if "." in current_module.__name__:
current_module = ".".join(current_module.__name__.split(".")[0:-1])
else:
current_module = ""
metadata.reverse()
return metadata
graphs
¶
print_ascii_graph(graph)
¶Source code in kiara/utils/graphs.py
def print_ascii_graph(graph: nx.Graph):
try:
from asciinet import graph_to_ascii
except: # noqa
print(
"\nCan't print graph on terminal, package 'asciinet' not available. Please install it into the current virtualenv using:\n\npip install 'git+https://github.com/cosminbasca/asciinet.git#egg=asciinet&subdirectory=pyasciinet'"
)
return
try:
from asciinet._libutil import check_java
check_java("Java ")
except Exception as e:
print(e)
print(
"\nJava is currently necessary to print ascii graph. This might change in the future, but to use this functionality please install a JRE."
)
return
print(graph_to_ascii(graph))
hashfs
special
¶
HashFS is a content-addressable file management system. What does that mean? Simply, that HashFS manages a directory where files are saved based on the file's hash.
Typical use cases for this kind of system are ones where:
- Files are written once and never change (e.g. image storage).
- It's desirable to have no duplicate files (e.g. user uploads).
- File metadata is stored elsewhere (e.g. in a database).
Adapted from: https://github.com/dgilland/hashfs
License¶
The MIT License (MIT)
Copyright (c) 2015, Derrick Gilland
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Classes¶
HashAddress (HashAddress)
¶File address containing file's path on disk and it's content hash ID.
Attributes:
| Name | Type | Description |
|---|---|---|
id |
str |
Hash ID (hexdigest) of file contents. |
relpath |
str |
Relative path location to :attr: |
abspath |
str |
Absoluate path location of file on disk. |
is_duplicate |
boolean |
Whether the hash address created was
a duplicate of a previously existing file. Can only be |
Source code in kiara/utils/hashfs/__init__.py
class HashAddress(
namedtuple("HashAddress", ["id", "relpath", "abspath", "is_duplicate"])
):
"""File address containing file's path on disk and it's content hash ID.
Attributes:
id (str): Hash ID (hexdigest) of file contents.
relpath (str): Relative path location to :attr:`HashFS.root`.
abspath (str): Absoluate path location of file on disk.
is_duplicate (boolean, optional): Whether the hash address created was
a duplicate of a previously existing file. Can only be ``True``
after a put operation. Defaults to ``False``.
"""
def __new__(cls, id, relpath, abspath, is_duplicate=False):
return super(HashAddress, cls).__new__(cls, id, relpath, abspath, is_duplicate) # type: ignore
HashFS
¶Content addressable file manager.
Attributes:
| Name | Type | Description |
|---|---|---|
root |
str |
Directory path used as root of storage space. |
depth |
int |
Depth of subfolders to create when saving a file. |
width |
int |
Width of each subfolder to create when saving a file. |
algorithm |
str |
Hash algorithm to use when computing file hash.
Algorithm should be available in |
fmode |
int |
File mode permission to set when adding files to
directory. Defaults to |
dmode |
int |
Directory mode permission to set for
subdirectories. Defaults to |
Source code in kiara/utils/hashfs/__init__.py
class HashFS(object):
"""Content addressable file manager.
Attributes:
root (str): Directory path used as root of storage space.
depth (int, optional): Depth of subfolders to create when saving a
file.
width (int, optional): Width of each subfolder to create when saving a
file.
algorithm (str): Hash algorithm to use when computing file hash.
Algorithm should be available in ``hashlib`` module. Defaults to
``'sha256'``.
fmode (int, optional): File mode permission to set when adding files to
directory. Defaults to ``0o664`` which allows owner/group to
read/write and everyone else to read.
dmode (int, optional): Directory mode permission to set for
subdirectories. Defaults to ``0o755`` which allows owner/group to
read/write and everyone else to read and everyone to execute.
"""
def __init__(
self,
root: str,
depth: int = 4,
width: int = 1,
algorithm: str = "sha256",
fmode=0o664,
dmode=0o755,
):
self.root: str = os.path.realpath(root)
self.depth: int = depth
self.width: int = width
self.algorithm: str = algorithm
self.fmode = fmode
self.dmode = dmode
def put(self, file: BinaryIO) -> "HashAddress":
"""Store contents of `file` on disk using its content hash for the
address.
Args:
file (mixed): Readable object or path to file.
Returns:
HashAddress: File's hash address.
"""
stream = Stream(file)
with closing(stream):
id = self.computehash(stream)
filepath, is_duplicate = self._copy(stream, id)
return HashAddress(id, self.relpath(filepath), filepath, is_duplicate)
def put_with_precomputed_hash(
self, file: Union[str, Path, BinaryIO], hash_id: str
) -> "HashAddress":
stream = Stream(file)
with closing(stream):
filepath, is_duplicate = self._copy(stream=stream, id=hash_id)
return HashAddress(hash_id, self.relpath(filepath), filepath, is_duplicate)
def _copy(self, stream: "Stream", id: str):
"""Copy the contents of `stream` onto disk with an optional file
extension appended. The copy process uses a temporary file to store the
initial contents and then moves that file to it's final location.
"""
filepath = self.idpath(id)
if not os.path.isfile(filepath):
# Only move file if it doesn't already exist.
is_duplicate = False
fname = self._mktempfile(stream)
self.makepath(os.path.dirname(filepath))
shutil.move(fname, filepath)
else:
is_duplicate = True
return (filepath, is_duplicate)
def _mktempfile(self, stream):
"""Create a named temporary file from a :class:`Stream` object and
return its filename.
"""
tmp = NamedTemporaryFile(delete=False)
if self.fmode is not None:
oldmask = os.umask(0)
try:
os.chmod(tmp.name, self.fmode)
finally:
os.umask(oldmask)
for data in stream:
tmp.write(to_bytes(data))
tmp.close()
return tmp.name
def get(self, file):
"""Return :class:`HashAdress` from given id or path. If `file` does not
refer to a valid file, then ``None`` is returned.
Args:
file (str): Address ID or path of file.
Returns:
HashAddress: File's hash address.
"""
realpath = self.realpath(file)
if realpath is None:
return None
else:
return HashAddress(self.unshard(realpath), self.relpath(realpath), realpath)
def open(self, file, mode="rb"):
"""Return open buffer object from given id or path.
Args:
file (str): Address ID or path of file.
mode (str, optional): Mode to open file in. Defaults to ``'rb'``.
Returns:
Buffer: An ``io`` buffer dependent on the `mode`.
Raises:
IOError: If file doesn't exist.
"""
realpath = self.realpath(file)
if realpath is None:
raise IOError("Could not locate file: {0}".format(file))
return io.open(realpath, mode)
def delete(self, file):
"""Delete file using id or path. Remove any empty directories after
deleting. No exception is raised if file doesn't exist.
Args:
file (str): Address ID or path of file.
"""
realpath = self.realpath(file)
if realpath is None:
return
try:
os.remove(realpath)
except OSError: # pragma: no cover
pass
else:
self.remove_empty(os.path.dirname(realpath))
def remove_empty(self, subpath):
"""Successively remove all empty folders starting with `subpath` and
proceeding "up" through directory tree until reaching the :attr:`root`
folder.
"""
# Don't attempt to remove any folders if subpath is not a
# subdirectory of the root directory.
if not self.haspath(subpath):
return
while subpath != self.root:
if len(os.listdir(subpath)) > 0 or os.path.islink(subpath):
break
os.rmdir(subpath)
subpath = os.path.dirname(subpath)
def files(self):
"""Return generator that yields all files in the :attr:`root`
directory.
"""
for folder, subfolders, files in walk(self.root):
for file in files:
yield os.path.abspath(os.path.join(folder, file))
def folders(self):
"""Return generator that yields all folders in the :attr:`root`
directory that contain files.
"""
for folder, subfolders, files in walk(self.root):
if files:
yield folder
def count(self):
"""Return count of the number of files in the :attr:`root` directory."""
count = 0
for _ in self:
count += 1
return count
def size(self):
"""Return the total size in bytes of all files in the :attr:`root`
directory.
"""
total = 0
for path in self.files():
total += os.path.getsize(path)
return total
def exists(self, file):
"""Check whether a given file id or path exists on disk."""
return bool(self.realpath(file))
def haspath(self, path):
"""Return whether `path` is a subdirectory of the :attr:`root`
directory.
"""
return issubdir(path, self.root)
def makepath(self, path):
"""Physically create the folder path on disk."""
try:
os.makedirs(path, self.dmode)
except FileExistsError:
assert os.path.isdir(path), "expected {} to be a directory".format(path)
def relpath(self, path):
"""Return `path` relative to the :attr:`root` directory."""
return os.path.relpath(path, self.root)
def realpath(self, file):
"""Attempt to determine the real path of a file id or path through
successive checking of candidate paths. If the real path is stored with
an extension, the path is considered a match if the basename matches
the expected file path of the id.
"""
# Check for absoluate path.
if os.path.isfile(file):
return file
# Check for relative path.
relpath = os.path.join(self.root, file)
if os.path.isfile(relpath):
return relpath
# Check for sharded path.
filepath = self.idpath(file)
if os.path.isfile(filepath):
return filepath
# Check for sharded path with any extension.
paths = glob.glob("{0}.*".format(filepath))
if paths:
return paths[0]
# Could not determine a match.
return None
def idpath(self, id):
"""Build the file path for a given hash id. Optionally, append a
file extension.
"""
paths = self.shard(id)
return os.path.join(self.root, *paths)
def computehash(self, stream) -> str:
"""Compute hash of file using :attr:`algorithm`."""
hashobj = hashlib.new(self.algorithm)
for data in stream:
hashobj.update(to_bytes(data))
return hashobj.hexdigest()
def shard(self, id):
"""Shard content ID into subfolders."""
return shard(id, self.depth, self.width)
def unshard(self, path):
"""Unshard path to determine hash value."""
if not self.haspath(path):
raise ValueError(
"Cannot unshard path. The path {0!r} is not "
"a subdirectory of the root directory {1!r}".format(path, self.root)
)
return os.path.splitext(self.relpath(path))[0].replace(os.sep, "")
def repair(self):
"""Repair any file locations whose content address doesn't match it's
file path.
"""
repaired = []
corrupted = tuple(self.corrupted())
oldmask = os.umask(0)
try:
for path, address in corrupted:
if os.path.isfile(address.abspath):
# File already exists so just delete corrupted path.
os.remove(path)
else:
# File doesn't exists so move it.
self.makepath(os.path.dirname(address.abspath))
shutil.move(path, address.abspath)
os.chmod(address.abspath, self.fmode)
repaired.append((path, address))
finally:
os.umask(oldmask)
return repaired
def corrupted(self):
"""Return generator that yields corrupted files as ``(path, address)``
where ``path`` is the path of the corrupted file and ``address`` is
the :class:`HashAddress` of the expected location.
"""
for path in self.files():
stream = Stream(path)
with closing(stream):
id = self.computehash(stream)
expected_path = self.idpath(id)
if expected_path != path:
yield (
path,
HashAddress(id, self.relpath(expected_path), expected_path),
)
def __contains__(self, file):
"""Return whether a given file id or path is contained in the
:attr:`root` directory.
"""
return self.exists(file)
def __iter__(self):
"""Iterate over all files in the :attr:`root` directory."""
return self.files()
def __len__(self):
"""Return count of the number of files in the :attr:`root` directory."""
return self.count()
computehash(self, stream)
¶Compute hash of file using :attr:algorithm.
Source code in kiara/utils/hashfs/__init__.py
def computehash(self, stream) -> str:
"""Compute hash of file using :attr:`algorithm`."""
hashobj = hashlib.new(self.algorithm)
for data in stream:
hashobj.update(to_bytes(data))
return hashobj.hexdigest()
corrupted(self)
¶Return generator that yields corrupted files as (path, address)
where path is the path of the corrupted file and address is
the :class:HashAddress of the expected location.
Source code in kiara/utils/hashfs/__init__.py
def corrupted(self):
"""Return generator that yields corrupted files as ``(path, address)``
where ``path`` is the path of the corrupted file and ``address`` is
the :class:`HashAddress` of the expected location.
"""
for path in self.files():
stream = Stream(path)
with closing(stream):
id = self.computehash(stream)
expected_path = self.idpath(id)
if expected_path != path:
yield (
path,
HashAddress(id, self.relpath(expected_path), expected_path),
)
count(self)
¶Return count of the number of files in the :attr:root directory.
Source code in kiara/utils/hashfs/__init__.py
def count(self):
"""Return count of the number of files in the :attr:`root` directory."""
count = 0
for _ in self:
count += 1
return count
delete(self, file)
¶Delete file using id or path. Remove any empty directories after deleting. No exception is raised if file doesn't exist.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
file |
str |
Address ID or path of file. |
required |
Source code in kiara/utils/hashfs/__init__.py
def delete(self, file):
"""Delete file using id or path. Remove any empty directories after
deleting. No exception is raised if file doesn't exist.
Args:
file (str): Address ID or path of file.
"""
realpath = self.realpath(file)
if realpath is None:
return
try:
os.remove(realpath)
except OSError: # pragma: no cover
pass
else:
self.remove_empty(os.path.dirname(realpath))
exists(self, file)
¶Check whether a given file id or path exists on disk.
Source code in kiara/utils/hashfs/__init__.py
def exists(self, file):
"""Check whether a given file id or path exists on disk."""
return bool(self.realpath(file))
files(self)
¶Return generator that yields all files in the :attr:root
directory.
Source code in kiara/utils/hashfs/__init__.py
def files(self):
"""Return generator that yields all files in the :attr:`root`
directory.
"""
for folder, subfolders, files in walk(self.root):
for file in files:
yield os.path.abspath(os.path.join(folder, file))
folders(self)
¶Return generator that yields all folders in the :attr:root
directory that contain files.
Source code in kiara/utils/hashfs/__init__.py
def folders(self):
"""Return generator that yields all folders in the :attr:`root`
directory that contain files.
"""
for folder, subfolders, files in walk(self.root):
if files:
yield folder
get(self, file)
¶Return :class:HashAdress from given id or path. If file does not
refer to a valid file, then None is returned.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
file |
str |
Address ID or path of file. |
required |
Returns:
| Type | Description |
|---|---|
HashAddress |
File's hash address. |
Source code in kiara/utils/hashfs/__init__.py
def get(self, file):
"""Return :class:`HashAdress` from given id or path. If `file` does not
refer to a valid file, then ``None`` is returned.
Args:
file (str): Address ID or path of file.
Returns:
HashAddress: File's hash address.
"""
realpath = self.realpath(file)
if realpath is None:
return None
else:
return HashAddress(self.unshard(realpath), self.relpath(realpath), realpath)
haspath(self, path)
¶Return whether path is a subdirectory of the :attr:root
directory.
Source code in kiara/utils/hashfs/__init__.py
def haspath(self, path):
"""Return whether `path` is a subdirectory of the :attr:`root`
directory.
"""
return issubdir(path, self.root)
idpath(self, id)
¶Build the file path for a given hash id. Optionally, append a file extension.
Source code in kiara/utils/hashfs/__init__.py
def idpath(self, id):
"""Build the file path for a given hash id. Optionally, append a
file extension.
"""
paths = self.shard(id)
return os.path.join(self.root, *paths)
makepath(self, path)
¶Physically create the folder path on disk.
Source code in kiara/utils/hashfs/__init__.py
def makepath(self, path):
"""Physically create the folder path on disk."""
try:
os.makedirs(path, self.dmode)
except FileExistsError:
assert os.path.isdir(path), "expected {} to be a directory".format(path)
open(self, file, mode='rb')
¶Return open buffer object from given id or path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
file |
str |
Address ID or path of file. |
required |
mode |
str |
Mode to open file in. Defaults to |
'rb' |
Returns:
| Type | Description |
|---|---|
Buffer |
An |
Exceptions:
| Type | Description |
|---|---|
IOError |
If file doesn't exist. |
Source code in kiara/utils/hashfs/__init__.py
def open(self, file, mode="rb"):
"""Return open buffer object from given id or path.
Args:
file (str): Address ID or path of file.
mode (str, optional): Mode to open file in. Defaults to ``'rb'``.
Returns:
Buffer: An ``io`` buffer dependent on the `mode`.
Raises:
IOError: If file doesn't exist.
"""
realpath = self.realpath(file)
if realpath is None:
raise IOError("Could not locate file: {0}".format(file))
return io.open(realpath, mode)
put(self, file)
¶Store contents of file on disk using its content hash for the
address.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
file |
mixed |
Readable object or path to file. |
required |
Returns:
| Type | Description |
|---|---|
HashAddress |
File's hash address. |
Source code in kiara/utils/hashfs/__init__.py
def put(self, file: BinaryIO) -> "HashAddress":
"""Store contents of `file` on disk using its content hash for the
address.
Args:
file (mixed): Readable object or path to file.
Returns:
HashAddress: File's hash address.
"""
stream = Stream(file)
with closing(stream):
id = self.computehash(stream)
filepath, is_duplicate = self._copy(stream, id)
return HashAddress(id, self.relpath(filepath), filepath, is_duplicate)
put_with_precomputed_hash(self, file, hash_id)
¶Source code in kiara/utils/hashfs/__init__.py
def put_with_precomputed_hash(
self, file: Union[str, Path, BinaryIO], hash_id: str
) -> "HashAddress":
stream = Stream(file)
with closing(stream):
filepath, is_duplicate = self._copy(stream=stream, id=hash_id)
return HashAddress(hash_id, self.relpath(filepath), filepath, is_duplicate)
realpath(self, file)
¶Attempt to determine the real path of a file id or path through successive checking of candidate paths. If the real path is stored with an extension, the path is considered a match if the basename matches the expected file path of the id.
Source code in kiara/utils/hashfs/__init__.py
def realpath(self, file):
"""Attempt to determine the real path of a file id or path through
successive checking of candidate paths. If the real path is stored with
an extension, the path is considered a match if the basename matches
the expected file path of the id.
"""
# Check for absoluate path.
if os.path.isfile(file):
return file
# Check for relative path.
relpath = os.path.join(self.root, file)
if os.path.isfile(relpath):
return relpath
# Check for sharded path.
filepath = self.idpath(file)
if os.path.isfile(filepath):
return filepath
# Check for sharded path with any extension.
paths = glob.glob("{0}.*".format(filepath))
if paths:
return paths[0]
# Could not determine a match.
return None
relpath(self, path)
¶Return path relative to the :attr:root directory.
Source code in kiara/utils/hashfs/__init__.py
def relpath(self, path):
"""Return `path` relative to the :attr:`root` directory."""
return os.path.relpath(path, self.root)
remove_empty(self, subpath)
¶Successively remove all empty folders starting with subpath and
proceeding "up" through directory tree until reaching the :attr:root
folder.
Source code in kiara/utils/hashfs/__init__.py
def remove_empty(self, subpath):
"""Successively remove all empty folders starting with `subpath` and
proceeding "up" through directory tree until reaching the :attr:`root`
folder.
"""
# Don't attempt to remove any folders if subpath is not a
# subdirectory of the root directory.
if not self.haspath(subpath):
return
while subpath != self.root:
if len(os.listdir(subpath)) > 0 or os.path.islink(subpath):
break
os.rmdir(subpath)
subpath = os.path.dirname(subpath)
repair(self)
¶Repair any file locations whose content address doesn't match it's file path.
Source code in kiara/utils/hashfs/__init__.py
def repair(self):
"""Repair any file locations whose content address doesn't match it's
file path.
"""
repaired = []
corrupted = tuple(self.corrupted())
oldmask = os.umask(0)
try:
for path, address in corrupted:
if os.path.isfile(address.abspath):
# File already exists so just delete corrupted path.
os.remove(path)
else:
# File doesn't exists so move it.
self.makepath(os.path.dirname(address.abspath))
shutil.move(path, address.abspath)
os.chmod(address.abspath, self.fmode)
repaired.append((path, address))
finally:
os.umask(oldmask)
return repaired
shard(self, id)
¶Shard content ID into subfolders.
Source code in kiara/utils/hashfs/__init__.py
def shard(self, id):
"""Shard content ID into subfolders."""
return shard(id, self.depth, self.width)
size(self)
¶Return the total size in bytes of all files in the :attr:root
directory.
Source code in kiara/utils/hashfs/__init__.py
def size(self):
"""Return the total size in bytes of all files in the :attr:`root`
directory.
"""
total = 0
for path in self.files():
total += os.path.getsize(path)
return total
unshard(self, path)
¶Unshard path to determine hash value.
Source code in kiara/utils/hashfs/__init__.py
def unshard(self, path):
"""Unshard path to determine hash value."""
if not self.haspath(path):
raise ValueError(
"Cannot unshard path. The path {0!r} is not "
"a subdirectory of the root directory {1!r}".format(path, self.root)
)
return os.path.splitext(self.relpath(path))[0].replace(os.sep, "")
Stream
¶Common interface for file-like objects.
The input obj can be a file-like object or a path to a file. If obj is
a path to a file, then it will be opened until :meth:close is called.
If obj is a file-like object, then it's original position will be
restored when :meth:close is called instead of closing the object
automatically. Closing of the stream is deferred to whatever process passed
the stream in.
Successive readings of the stream is supported without having to manually
set it's position back to 0.
Source code in kiara/utils/hashfs/__init__.py
class Stream(object):
"""Common interface for file-like objects.
The input `obj` can be a file-like object or a path to a file. If `obj` is
a path to a file, then it will be opened until :meth:`close` is called.
If `obj` is a file-like object, then it's original position will be
restored when :meth:`close` is called instead of closing the object
automatically. Closing of the stream is deferred to whatever process passed
the stream in.
Successive readings of the stream is supported without having to manually
set it's position back to ``0``.
"""
def __init__(self, obj: Union[BinaryIO, str, Path]):
if hasattr(obj, "read"):
pos = obj.tell() # type: ignore
elif os.path.isfile(obj): # type: ignore
obj = io.open(obj, "rb") # type: ignore
pos = None
else:
raise ValueError("Object must be a valid file path or a readable object")
try:
file_stat = os.stat(obj.name) # type: ignore
buffer_size = file_stat.st_blksize
except Exception:
buffer_size = 8192
self._obj = obj
self._pos = pos
self._buffer_size = buffer_size
def __iter__(self):
"""Read underlying IO object and yield results. Return object to
original position if we didn't open it originally.
"""
self._obj.seek(0)
while True:
data = self._obj.read(self._buffer_size)
if not data:
break
yield data
if self._pos is not None:
self._obj.seek(self._pos)
def close(self):
"""Close underlying IO object if we opened it, else return it to
original position.
"""
if self._pos is None:
self._obj.close()
else:
self._obj.seek(self._pos)
close(self)
¶Close underlying IO object if we opened it, else return it to original position.
Source code in kiara/utils/hashfs/__init__.py
def close(self):
"""Close underlying IO object if we opened it, else return it to
original position.
"""
if self._pos is None:
self._obj.close()
else:
self._obj.seek(self._pos)
Functions¶
compact(items)
¶Return only truthy elements of items.
Source code in kiara/utils/hashfs/__init__.py
def compact(items: List[Any]) -> List[Any]:
"""Return only truthy elements of `items`."""
return [item for item in items if item]
issubdir(subpath, path)
¶Return whether subpath is a sub-directory of path.
Source code in kiara/utils/hashfs/__init__.py
def issubdir(subpath: str, path: str):
"""Return whether `subpath` is a sub-directory of `path`."""
# Append os.sep so that paths like /usr/var2/log doesn't match /usr/var.
path = os.path.realpath(path) + os.sep
subpath = os.path.realpath(subpath)
return subpath.startswith(path)
shard(digest, depth, width)
¶Source code in kiara/utils/hashfs/__init__.py
def shard(digest, depth, width):
# This creates a list of `depth` number of tokens with width
# `width` from the first part of the id plus the remainder.
return compact(
[digest[i * width : width * (i + 1)] for i in range(depth)] # noqa
+ [digest[depth * width :]] # noqa
)
to_bytes(text)
¶Source code in kiara/utils/hashfs/__init__.py
def to_bytes(text: Union[str, bytes]):
if not isinstance(text, bytes):
text = bytes(text, "utf8")
return text
hashing
¶
NONE_CID
¶compute_cid(data, hash_codec='sha2-256', encode='base58btc')
¶Source code in kiara/utils/hashing.py
def compute_cid(
data: EncodableType,
hash_codec: str = "sha2-256",
encode: str = "base58btc",
) -> Tuple[bytes, CID]:
encoded = dag_cbor.encode(data)
hash_func = multihash.get(hash_codec)
digest = hash_func.digest(encoded)
cid = CID(encode, 1, codec="dag-cbor", digest=digest)
return encoded, cid
compute_cid_from_file(file, codec='raw', hash_codec='sha2-256')
¶Source code in kiara/utils/hashing.py
def compute_cid_from_file(
file: str, codec: Union[str, int, Multicodec] = "raw", hash_codec: str = "sha2-256"
):
assert hash_codec == "sha2-256"
hash_func = hashlib.sha256
file_hash = hash_func()
CHUNK_SIZE = 65536
with open(file, "rb") as f:
fb = f.read(CHUNK_SIZE)
while len(fb) > 0:
file_hash.update(fb)
fb = f.read(CHUNK_SIZE)
wrapped = multihash.wrap(file_hash.digest(), "sha2-256")
return create_cid_digest(digest=wrapped, codec=codec)
create_cid_digest(digest, codec='raw')
¶Source code in kiara/utils/hashing.py
def create_cid_digest(
digest: Union[
str, BytesLike, Tuple[Union[str, int, Multihash], Union[str, BytesLike]]
],
codec: Union[str, int, Multicodec] = "raw",
) -> CID:
cid = CID("base58btc", 1, codec, digest)
return cid
html
¶
Functions¶
generate_html(item, render_config=None, add_header=False, add_type_column=False)
¶Create html representing this models data.
Source code in kiara/utils/html.py
def generate_html(
item: Any,
render_config: Union[None, Mapping[str, Any]] = None,
add_header: bool = False,
add_type_column: bool = False,
) -> Airium:
"""Create html representing this models data."""
doc = Airium()
if render_config is None:
render_config = {}
else:
render_config = dict(render_config)
if isinstance(item, str):
doc(item)
elif isinstance(item, BaseModel):
from kiara.models import KiaraModel
if isinstance(item, KiaraModel):
template_registry = TemplateRegistry.instance()
template = template_registry.get_template_for_model_type(
model_type=item.model_type_id, template_format="html"
)
if template:
rendered = template.render(instance=item)
doc(rendered)
return doc
exclude_fields = None
model_cls = item.__class__
props = model_cls.schema().get("properties", {})
rows = []
for field_name, field in model_cls.__fields__.items():
if exclude_fields and field_name in exclude_fields:
continue
row = [field_name]
p = props.get(field_name, None)
if add_type_column:
p_type = None
if p is not None:
p_type = p.get("type", None)
# TODO: check 'anyOf' keys
if p_type is None:
p_type = "-- check source --"
row.append(p_type)
data = getattr(item, field_name)
row.append(generate_html(data, render_config=render_config))
desc = p.get("description", "")
row.append(desc)
rows.append(row)
with doc.table():
if add_header:
with doc.tr():
doc.th(_t="field")
if add_type_column:
doc.th(_t="type")
doc.th(_t="data")
doc.th(_t="description")
for row in rows:
with doc.tr():
doc.td(_t=row[0])
doc.td(_t=row[1])
doc.td(_t=row[2])
if add_type_column:
doc.td(_t=row[3])
elif isinstance(item, Mapping):
with doc.table():
for k, v in item.items():
with doc.tr():
doc.td(_t=k)
value_el = generate_html(v)
doc.td(_t=value_el)
elif isinstance(item, Iterable):
with doc.ul():
for i in item:
with doc.li():
value_el = generate_html(i)
doc(str(value_el))
else:
doc(str(item))
return doc
json
¶
orjson_dumps(v, *, default=None, **args)
¶Source code in kiara/utils/json.py
def orjson_dumps(v, *, default=None, **args):
# orjson.dumps returns bytes, to match standard json.dumps we need to decode
try:
return orjson.dumps(v, default=default, **args).decode()
except Exception as e:
if is_debug():
print(f"Error dumping json data: {e}")
from kiara import dbg
dbg(v)
raise e
jupyter
¶
create_image(graph)
¶Source code in kiara/utils/jupyter.py
def create_image(graph: nx.Graph):
try:
import pygraphviz as pgv # noqa
except: # noqa
return "pygraphviz not available, please install it manually into the current virtualenv"
# graph = nx.relabel_nodes(graph, lambda x: hash(x))
G = nx.nx_agraph.to_agraph(graph)
G.node_attr["shape"] = "box"
# G.unflatten().layout(prog="dot")
G.layout(prog="dot")
b = G.draw(format="png")
return b
graph_to_image(graph, return_bytes=False)
¶Source code in kiara/utils/jupyter.py
def graph_to_image(graph: nx.Graph, return_bytes: bool = False):
b = create_image(graph=graph)
if return_bytes:
return b
else:
from IPython.core.display import Image
return Image(b)
save_image(graph, path)
¶Source code in kiara/utils/jupyter.py
def save_image(graph: nx.Graph, path: str):
graph_b = create_image(graph=graph)
with open(path, "wb") as f:
f.write(graph_b)
metadata
¶
find_metadata_models(alias=None, only_for_package=None)
¶Source code in kiara/utils/metadata.py
def find_metadata_models(
alias: Union[str, None] = None, only_for_package: Union[str, None] = None
) -> MetadataTypeClassesInfo:
model_registry = ModelRegistry.instance()
_group = model_registry.get_models_of_type(ValueMetadata) # type: ignore
classes: Dict[str, Type[ValueMetadata]] = {}
for model_id, info in _group.item_infos.items():
classes[model_id] = info.python_class.get_class() # type: ignore
group: MetadataTypeClassesInfo = MetadataTypeClassesInfo.create_from_type_items(group_title=alias, kiara=None, **classes) # type: ignore
if only_for_package:
temp = {}
for key, _info in group.item_infos.items():
if _info.context.labels.get("package") == only_for_package:
temp[key] = _info
group = MetadataTypeClassesInfo.construct(
group_id=group.instance_id, group_alias=group.group_alias, item_infos=temp # type: ignore
)
return group
models
¶
Functions¶
assemble_subcomponent_graph(data)
¶Source code in kiara/utils/models.py
def assemble_subcomponent_graph(data: "KiaraModel") -> Union[nx.DiGraph, None]:
from kiara.models import KiaraModel
graph = nx.DiGraph()
def assemble_graph(info_model: KiaraModel, current_node_id, level: int = 0):
graph.add_node(current_node_id, obj=info_model, level=level)
scn = info_model.subcomponent_keys
if not scn:
return
for child_path in scn:
child_obj = info_model.get_subcomponent(child_path)
new_node_id = f"{current_node_id}.{child_path}"
graph.add_edge(current_node_id, new_node_id)
if isinstance(child_obj, KiaraModel):
assemble_graph(child_obj, new_node_id, level + 1)
assemble_graph(data, KIARA_DEFAULT_ROOT_NODE_ID)
return graph
create_pydantic_model(model_cls, _use_pydantic_construct=False, **field_values)
¶Source code in kiara/utils/models.py
def create_pydantic_model(
model_cls: Type[BaseModel],
_use_pydantic_construct: bool = PYDANTIC_USE_CONSTRUCT,
**field_values: Any,
):
if _use_pydantic_construct:
raise NotImplementedError()
return model_cls.construct(**field_values)
else:
return model_cls(**field_values)
create_subcomponent_tree_renderable(data, show_data=False)
¶Source code in kiara/utils/models.py
def create_subcomponent_tree_renderable(
data: "KiaraModel", show_data: bool = False
) -> Tree:
from kiara.models import KiaraModel
from kiara.utils.output import extract_renderable
def extract_type_string(obj: Any) -> str:
if isinstance(obj, KiaraModel):
return f"model: {obj.model_type_id}"
elif isinstance(obj, Mapping):
return "dict"
else:
return type(obj).__name__
def assemble_tree(node: Tree, model: Any, level: int):
if isinstance(model, Mapping) and model:
for k, v in model.items():
child_tree = node.add(f"[b i]{k}[/b i] ({extract_type_string(v)})")
assemble_tree(node=child_tree, model=v, level=level + 1)
return
if not isinstance(model, KiaraModel):
if show_data:
renderable = extract_renderable(model)
panel = Panel(
renderable, title="[i]data[/i]", title_align="left", expand=False
)
node.add(panel)
return
scn = model.subcomponent_keys
if not scn:
return
for child_path in scn:
child_obj = model.get_subcomponent(child_path)
child_tree = node.add(
f"[b i]{child_path}[/b i] ({extract_type_string(child_obj)})"
)
assemble_tree(node=child_tree, model=child_obj, level=level + 1)
tree = Tree(f"[b]{data.model_type_id}[/b]: [b]{data.instance_id}[/b]")
assemble_tree(node=tree, model=data, level=0)
return tree
get_subcomponent_from_model(data, path)
¶Return subcomponents of a model under a specified path.
Source code in kiara/utils/models.py
def get_subcomponent_from_model(data: "KiaraModel", path: str) -> "KiaraModel":
"""Return subcomponents of a model under a specified path."""
if "." in path:
first_token, rest = path.split(".", maxsplit=1)
sc = data.get_subcomponent(first_token)
return sc.get_subcomponent(rest)
if hasattr(data, "__custom_root_type__") and data.__custom_root_type__:
if isinstance(data.__root__, Mapping): # type: ignore
if path in data.__root__.keys(): # type: ignore
return data.__root__[path] # type: ignore
else:
matches = {}
for k in data.__root__.keys(): # type: ignore
if k.startswith(f"{path}."):
rest = k[len(path) + 1 :] # noqa
matches[rest] = data.__root__[k] # type: ignore
if not matches:
raise KeyError(f"No child models under '{path}'.")
else:
raise NotImplementedError()
# subcomponent_group = KiaraModelGroup.create_from_child_models(**matches)
# return subcomponent_group
else:
raise NotImplementedError()
else:
if path in data.__fields__.keys():
return getattr(data, path)
else:
raise KeyError(
f"No subcomponent for key '{path}' in model: {data.instance_id}."
)
retrieve_data_subcomponent_keys(data)
¶Source code in kiara/utils/models.py
def retrieve_data_subcomponent_keys(data: Any) -> Iterable[str]:
if hasattr(data, "__custom_root_type__") and data.__custom_root_type__:
if isinstance(data.__root__, Mapping): # type: ignore
result = set()
for k, v in data.__root__.items(): # type: ignore
if isinstance(v, BaseModel):
result.add(k.split(".")[0])
return result
else:
return []
elif isinstance(data, BaseModel):
matches = sorted(data.__fields__.keys())
return matches
else:
log_message(
f"No subcomponents retrieval supported for data of type: {type(data)}"
)
return []
modules
¶
module_config_is_empty(config)
¶Source code in kiara/utils/modules.py
def module_config_is_empty(config: Mapping[str, Any]):
c = dict(config)
d = c.pop("defaults", None)
if d:
return False
constants = c.pop("constants", None)
if constants:
return False
return False if c else True
operations
¶
create_operation(module_or_operation, operation_config=None, kiara=None)
¶Source code in kiara/utils/operations.py
def create_operation(
module_or_operation: str,
operation_config: Union[None, Mapping[str, Any]] = None,
kiara: Union[None, "Kiara"] = None,
) -> Operation:
operation: Union[Operation, None]
if kiara is None:
from kiara.context import Kiara
kiara = Kiara.instance()
if module_or_operation in kiara.operation_registry.operation_ids:
operation = kiara.operation_registry.get_operation(module_or_operation)
if operation_config:
raise Exception(
f"Specified run target '{module_or_operation}' is an operation, additional module configuration is not allowed."
)
elif module_or_operation in kiara.module_type_names:
manifest = Manifest(
module_type=module_or_operation, module_config=operation_config
)
module = kiara.create_module(manifest=manifest)
operation = Operation.create_from_module(module)
elif os.path.isfile(module_or_operation):
data = get_data_from_file(module_or_operation)
pipeline_name = data.pop("pipeline_name", None)
if pipeline_name is None:
pipeline_name = os.path.basename(module_or_operation)
# self._defaults = data.pop("inputs", {})
execution_context = ExecutionContext(
pipeline_dir=os.path.abspath(os.path.dirname(module_or_operation))
)
pipeline_config = PipelineConfig.from_config(
pipeline_name=pipeline_name,
data=data,
kiara=kiara,
execution_context=execution_context,
)
manifest = kiara.create_manifest("pipeline", config=pipeline_config.dict())
module = kiara.create_module(manifest=manifest)
operation = Operation.create_from_module(module, doc=pipeline_config.doc)
else:
raise Exception(
f"Can't assemble operation, invalid operation/module name: {module_or_operation}. Must be registered module or operation name, or file."
)
# manifest = Manifest(
# module_type=module_or_operation,
# module_config=self._operation_config,
# )
# module = self._kiara.create_module(manifest=manifest)
# operation = Operation.create_from_module(module=module)
if operation is None:
merged = set(kiara.module_type_names)
merged.update(kiara.operation_registry.operation_ids)
raise NoSuchExecutionTargetException(
selected_target=module_or_operation,
msg=f"Invalid run target name '{module_or_operation}'. Must be a path to a pipeline file, or one of the available modules/operations.",
available_targets=sorted(merged),
)
return operation
filter_operations(kiara, pkg_name=None, **operations)
¶Source code in kiara/utils/operations.py
def filter_operations(
kiara: "Kiara", pkg_name: Union[str, None] = None, **operations: "Operation"
) -> OperationGroupInfo:
result: Dict[str, OperationInfo] = {}
# op_infos = kiara.operation_registry.get_context_metadata(only_for_package=pkg_name)
modules = kiara.module_registry.get_context_metadata(only_for_package=pkg_name)
for op_id, op in operations.items():
if op.module.module_type_name != "pipeline":
if op.module.module_type_name in modules.item_infos.keys():
result[op_id] = OperationInfo.create_from_operation(
kiara=kiara, operation=op
)
continue
else:
package: Union[str, None] = op.metadata.get("labels", {}).get(
"package", None
)
if not pkg_name or (package and package == pkg_name):
result[op_id] = OperationInfo.create_from_operation(
kiara=kiara, operation=op
)
# opt_types = kiara.operation_registry.find_all_operation_types(op_id)
# match = False
# for ot in opt_types:
# if ot in op_infos.keys():
# match = True
# break
#
# if match:
# result[op_id] = OperationInfo.create_from_operation(
# kiara=kiara, operation=op
# )
return OperationGroupInfo.construct(item_infos=result) # type: ignore
output
¶
log
¶Classes¶
ArrowTabularWrap (TabularWrap)
¶Source code in kiara/utils/output.py
class ArrowTabularWrap(TabularWrap):
def __init__(self, table: "ArrowTable"):
self._table: "ArrowTable" = table
super().__init__()
def retrieve_column_names(self) -> Iterable[str]:
return self._table.column_names
def retrieve_number_of_rows(self) -> int:
return self._table.num_rows
def slice(self, offset: int = 0, length: Union[int, None] = None):
return self._table.slice(offset=offset, length=length)
def to_pydict(self) -> Mapping:
return self._table.to_pydict()
retrieve_column_names(self)
¶Source code in kiara/utils/output.py
def retrieve_column_names(self) -> Iterable[str]:
return self._table.column_names
retrieve_number_of_rows(self)
¶Source code in kiara/utils/output.py
def retrieve_number_of_rows(self) -> int:
return self._table.num_rows
slice(self, offset=0, length=None)
¶Source code in kiara/utils/output.py
def slice(self, offset: int = 0, length: Union[int, None] = None):
return self._table.slice(offset=offset, length=length)
to_pydict(self)
¶Source code in kiara/utils/output.py
def to_pydict(self) -> Mapping:
return self._table.to_pydict()
DictTabularWrap (TabularWrap)
¶Source code in kiara/utils/output.py
class DictTabularWrap(TabularWrap):
def __init__(self, data: Mapping[str, List[Any]]):
self._data: Mapping[str, List[Any]] = data
# TODO: assert all rows are equal length
super().__init__()
def retrieve_number_of_rows(self) -> int:
key = next(iter(self._data.keys()))
return len(self._data[key])
def retrieve_column_names(self) -> Iterable[str]:
return self._data.keys()
def to_pydict(self) -> Mapping[str, List[Any]]:
return self._data
def slice(self, offset: int = 0, length: Union[int, None] = None) -> "TabularWrap":
result = {}
start = None
end = None
for cn in self._data.keys():
if start is None:
if offset > len(self._data):
return DictTabularWrap({cn: [] for cn in self._data.keys()})
start = offset
if not length:
end = len(self._data)
else:
end = start + length
if end > len(self._data):
end = len(self._data)
result[cn] = self._data[cn][start:end]
return DictTabularWrap(result)
retrieve_column_names(self)
¶Source code in kiara/utils/output.py
def retrieve_column_names(self) -> Iterable[str]:
return self._data.keys()
retrieve_number_of_rows(self)
¶Source code in kiara/utils/output.py
def retrieve_number_of_rows(self) -> int:
key = next(iter(self._data.keys()))
return len(self._data[key])
slice(self, offset=0, length=None)
¶Source code in kiara/utils/output.py
def slice(self, offset: int = 0, length: Union[int, None] = None) -> "TabularWrap":
result = {}
start = None
end = None
for cn in self._data.keys():
if start is None:
if offset > len(self._data):
return DictTabularWrap({cn: [] for cn in self._data.keys()})
start = offset
if not length:
end = len(self._data)
else:
end = start + length
if end > len(self._data):
end = len(self._data)
result[cn] = self._data[cn][start:end]
return DictTabularWrap(result)
to_pydict(self)
¶Source code in kiara/utils/output.py
def to_pydict(self) -> Mapping[str, List[Any]]:
return self._data
OutputDetails (BaseModel)
pydantic-model
¶Source code in kiara/utils/output.py
class OutputDetails(BaseModel):
@classmethod
def from_data(cls, data: Any):
if isinstance(data, str):
if "=" in data:
data = [data]
else:
data = [f"format={data}"]
if isinstance(data, Iterable):
from kiara.utils.cli import dict_from_cli_args
data = list(data)
if len(data) == 1 and isinstance(data[0], str) and "=" not in data[0]:
data = [f"format={data[0]}"]
output_details_dict = dict_from_cli_args(*data)
else:
raise TypeError(
f"Can't parse output detail config: invalid input type '{type(data)}'."
)
output_details = OutputDetails(**output_details_dict)
return output_details
format: str = Field(description="The output format.")
target: str = Field(description="The output target.")
config: Dict[str, Any] = Field(
description="Output configuration.", default_factory=dict
)
@root_validator(pre=True)
def _set_defaults(cls, values):
target: str = values.pop("target", "terminal")
format: str = values.pop("format", None)
if format is None:
if target == "terminal":
format = "terminal"
else:
if target == "file":
format = "json"
else:
ext = target.split(".")[-1]
if ext in ["yaml", "json"]:
format = ext
else:
format = "json"
result = {"format": format, "target": target, "config": dict(values)}
return result
config: Dict[str, Any]
pydantic-field
¶Output configuration.
format: str
pydantic-field
required
¶The output format.
target: str
pydantic-field
required
¶The output target.
from_data(data)
classmethod
¶Source code in kiara/utils/output.py
@classmethod
def from_data(cls, data: Any):
if isinstance(data, str):
if "=" in data:
data = [data]
else:
data = [f"format={data}"]
if isinstance(data, Iterable):
from kiara.utils.cli import dict_from_cli_args
data = list(data)
if len(data) == 1 and isinstance(data[0], str) and "=" not in data[0]:
data = [f"format={data[0]}"]
output_details_dict = dict_from_cli_args(*data)
else:
raise TypeError(
f"Can't parse output detail config: invalid input type '{type(data)}'."
)
output_details = OutputDetails(**output_details_dict)
return output_details
RenderConfig (BaseModel)
pydantic-model
¶
SqliteTabularWrap (TabularWrap)
¶Source code in kiara/utils/output.py
class SqliteTabularWrap(TabularWrap):
def __init__(self, engine: "Engine", table_name: str):
self._engine: Engine = engine
self._table_name: str = table_name
super().__init__()
def retrieve_number_of_rows(self) -> int:
from sqlalchemy import text
with self._engine.connect() as con:
result = con.execute(text(f"SELECT count(*) from {self._table_name}"))
num_rows = result.fetchone()[0]
return num_rows
def retrieve_column_names(self) -> Iterable[str]:
from sqlalchemy import inspect
engine = self._engine
inspector = inspect(engine)
columns = inspector.get_columns(self._table_name)
result = [column["name"] for column in columns]
return result
def slice(self, offset: int = 0, length: Union[int, None] = None) -> "TabularWrap":
from sqlalchemy import text
query = f"SELECT * FROM {self._table_name}"
if length:
query = f"{query} LIMIT {length}"
else:
query = f"{query} LIMIT {self.num_rows}"
if offset > 0:
query = f"{query} OFFSET {offset}"
with self._engine.connect() as con:
result = con.execute(text(query))
result_dict: Dict[str, List[Any]] = {}
for cn in self.column_names:
result_dict[cn] = []
for r in result:
for i, cn in enumerate(self.column_names):
result_dict[cn].append(r[i])
return DictTabularWrap(result_dict)
def to_pydict(self) -> Mapping:
from sqlalchemy import text
query = f"SELECT * FROM {self._table_name}"
with self._engine.connect() as con:
result = con.execute(text(query))
result_dict: Dict[str, List[Any]] = {}
for cn in self.column_names:
result_dict[cn] = []
for r in result:
for i, cn in enumerate(self.column_names):
result_dict[cn].append(r[i])
return result_dict
retrieve_column_names(self)
¶Source code in kiara/utils/output.py
def retrieve_column_names(self) -> Iterable[str]:
from sqlalchemy import inspect
engine = self._engine
inspector = inspect(engine)
columns = inspector.get_columns(self._table_name)
result = [column["name"] for column in columns]
return result
retrieve_number_of_rows(self)
¶Source code in kiara/utils/output.py
def retrieve_number_of_rows(self) -> int:
from sqlalchemy import text
with self._engine.connect() as con:
result = con.execute(text(f"SELECT count(*) from {self._table_name}"))
num_rows = result.fetchone()[0]
return num_rows
slice(self, offset=0, length=None)
¶Source code in kiara/utils/output.py
def slice(self, offset: int = 0, length: Union[int, None] = None) -> "TabularWrap":
from sqlalchemy import text
query = f"SELECT * FROM {self._table_name}"
if length:
query = f"{query} LIMIT {length}"
else:
query = f"{query} LIMIT {self.num_rows}"
if offset > 0:
query = f"{query} OFFSET {offset}"
with self._engine.connect() as con:
result = con.execute(text(query))
result_dict: Dict[str, List[Any]] = {}
for cn in self.column_names:
result_dict[cn] = []
for r in result:
for i, cn in enumerate(self.column_names):
result_dict[cn].append(r[i])
return DictTabularWrap(result_dict)
to_pydict(self)
¶Source code in kiara/utils/output.py
def to_pydict(self) -> Mapping:
from sqlalchemy import text
query = f"SELECT * FROM {self._table_name}"
with self._engine.connect() as con:
result = con.execute(text(query))
result_dict: Dict[str, List[Any]] = {}
for cn in self.column_names:
result_dict[cn] = []
for r in result:
for i, cn in enumerate(self.column_names):
result_dict[cn].append(r[i])
return result_dict
TabularWrap (ABC)
¶Source code in kiara/utils/output.py
class TabularWrap(ABC):
def __init__(self):
self._num_rows: Union[int, None] = None
self._column_names: Union[Iterable[str], None] = None
self._force_single_line: bool = True
@property
def num_rows(self) -> int:
if self._num_rows is None:
self._num_rows = self.retrieve_number_of_rows()
return self._num_rows
@property
def column_names(self) -> Iterable[str]:
if self._column_names is None:
self._column_names = self.retrieve_column_names()
return self._column_names
@abstractmethod
def retrieve_column_names(self) -> Iterable[str]:
pass
@abstractmethod
def retrieve_number_of_rows(self) -> int:
pass
@abstractmethod
def slice(self, offset: int = 0, length: Union[int, None] = None) -> "TabularWrap":
pass
@abstractmethod
def to_pydict(self) -> Mapping:
pass
def as_string(
self,
rows_head: Union[int, None] = None,
rows_tail: Union[int, None] = None,
max_row_height: Union[int, None] = None,
max_cell_length: Union[int, None] = None,
):
table_str = ""
for cn in self.column_names:
table_str = f"{table_str}{cn}\t"
table_str = f"{table_str}\n"
for data in self.prepare_table_data(
return_column_names=False,
rows_head=rows_head,
rows_tail=rows_tail,
max_row_height=max_row_height,
max_cell_length=max_cell_length,
):
for cell in data:
table_str = f"{table_str}{cell}\t"
table_str = f"{table_str}\n"
return table_str
def as_html(
self,
rows_head: Union[int, None] = None,
rows_tail: Union[int, None] = None,
max_row_height: Union[int, None] = None,
max_cell_length: Union[int, None] = None,
) -> str:
table_str = "<table><tr>"
for cn in self.column_names:
table_str = f"{table_str}<th>{cn}</th>"
table_str = f"{table_str}</tr>"
for data in self.prepare_table_data(
return_column_names=False,
rows_head=rows_head,
rows_tail=rows_tail,
max_row_height=max_row_height,
max_cell_length=max_cell_length,
):
table_str = f"{table_str}<tr>"
for cell in data:
table_str = f"{table_str}<td>{cell}</td>"
table_str = f"{table_str}</tr>"
table_str = f"{table_str}</table>"
return table_str
def as_terminal_renderable(
self,
rows_head: Union[int, None] = None,
rows_tail: Union[int, None] = None,
max_row_height: Union[int, None] = None,
max_cell_length: Union[int, None] = None,
show_table_header: bool = True,
) -> RichTable:
rich_table = RichTable(show_header=show_table_header, box=box.SIMPLE)
if max_row_height == 1:
overflow = "ignore"
else:
overflow = "ellipsis"
for cn in self.column_names:
rich_table.add_column(cn, overflow=overflow) # type: ignore
data = self.prepare_table_data(
return_column_names=False,
rows_head=rows_head,
rows_tail=rows_tail,
max_row_height=max_row_height,
max_cell_length=max_cell_length,
)
for row in data:
rich_table.add_row(*row)
return rich_table
def prepare_table_data(
self,
return_column_names: bool = False,
rows_head: Union[int, None] = None,
rows_tail: Union[int, None] = None,
max_row_height: Union[int, None] = None,
max_cell_length: Union[int, None] = None,
) -> Iterator[Iterable[Any]]:
if return_column_names:
yield self.column_names
num_split_rows = 2
if rows_head is not None:
if rows_head < 0:
rows_head = 0
if rows_head > self.num_rows:
rows_head = self.num_rows
rows_tail = None
num_split_rows = 0
if rows_tail is not None:
if rows_head + rows_tail >= self.num_rows: # type: ignore
rows_head = self.num_rows
rows_tail = None
num_split_rows = 0
else:
num_split_rows = 0
if rows_head is not None:
head = self.slice(0, rows_head)
num_rows = rows_head
else:
head = self
num_rows = self.num_rows
table_dict = head.to_pydict()
for i in range(0, num_rows):
row = []
for cn in self.column_names:
cell = table_dict[cn][i]
cell_str = str(cell)
if max_row_height and max_row_height > 0 and "\n" in cell_str:
lines = cell_str.split("\n")
if len(lines) > max_row_height:
if max_row_height == 1:
lines = lines[0:1]
else:
half = int(max_row_height / 2)
lines = lines[0:half] + [".."] + lines[-half:]
cell_str = "\n".join(lines)
if max_cell_length and max_cell_length > 0:
lines = []
for line in cell_str.split("\n"):
if len(line) > max_cell_length:
line = line[0:max_cell_length] + " ..."
else:
line = line
lines.append(line)
cell_str = "\n".join(lines)
row.append(cell_str)
yield row
if num_split_rows:
for i in range(0, num_split_rows):
row = []
for _ in self.column_names:
row.append("...")
yield row
if rows_head:
if rows_tail is not None:
if rows_tail < 0:
rows_tail = 0
tail = self.slice(self.num_rows - rows_tail)
table_dict = tail.to_pydict()
for i in range(0, num_rows):
row = []
for cn in self.column_names:
cell = table_dict[cn][i]
cell_str = str(cell)
if max_row_height and max_row_height > 0 and "\n" in cell_str:
lines = cell_str.split("\n")
if len(lines) > max_row_height:
if max_row_height == 1:
lines = lines[0:1]
else:
half = int(len(lines) / 2)
lines = lines[0:half] + [".."] + lines[-half:]
cell_str = "\n".join(lines)
if max_cell_length and max_cell_length > 0:
lines = []
for line in cell_str.split("\n"):
if len(line) > max_cell_length:
line = line[0:(max_cell_length)] + " ..."
else:
line = line
lines.append(line)
cell_str = "\n".join(lines)
row.append(cell_str)
yield row
return
column_names: Iterable[str]
property
readonly
¶num_rows: int
property
readonly
¶as_html(self, rows_head=None, rows_tail=None, max_row_height=None, max_cell_length=None)
¶Source code in kiara/utils/output.py
def as_html(
self,
rows_head: Union[int, None] = None,
rows_tail: Union[int, None] = None,
max_row_height: Union[int, None] = None,
max_cell_length: Union[int, None] = None,
) -> str:
table_str = "<table><tr>"
for cn in self.column_names:
table_str = f"{table_str}<th>{cn}</th>"
table_str = f"{table_str}</tr>"
for data in self.prepare_table_data(
return_column_names=False,
rows_head=rows_head,
rows_tail=rows_tail,
max_row_height=max_row_height,
max_cell_length=max_cell_length,
):
table_str = f"{table_str}<tr>"
for cell in data:
table_str = f"{table_str}<td>{cell}</td>"
table_str = f"{table_str}</tr>"
table_str = f"{table_str}</table>"
return table_str
as_string(self, rows_head=None, rows_tail=None, max_row_height=None, max_cell_length=None)
¶Source code in kiara/utils/output.py
def as_string(
self,
rows_head: Union[int, None] = None,
rows_tail: Union[int, None] = None,
max_row_height: Union[int, None] = None,
max_cell_length: Union[int, None] = None,
):
table_str = ""
for cn in self.column_names:
table_str = f"{table_str}{cn}\t"
table_str = f"{table_str}\n"
for data in self.prepare_table_data(
return_column_names=False,
rows_head=rows_head,
rows_tail=rows_tail,
max_row_height=max_row_height,
max_cell_length=max_cell_length,
):
for cell in data:
table_str = f"{table_str}{cell}\t"
table_str = f"{table_str}\n"
return table_str
as_terminal_renderable(self, rows_head=None, rows_tail=None, max_row_height=None, max_cell_length=None, show_table_header=True)
¶Source code in kiara/utils/output.py
def as_terminal_renderable(
self,
rows_head: Union[int, None] = None,
rows_tail: Union[int, None] = None,
max_row_height: Union[int, None] = None,
max_cell_length: Union[int, None] = None,
show_table_header: bool = True,
) -> RichTable:
rich_table = RichTable(show_header=show_table_header, box=box.SIMPLE)
if max_row_height == 1:
overflow = "ignore"
else:
overflow = "ellipsis"
for cn in self.column_names:
rich_table.add_column(cn, overflow=overflow) # type: ignore
data = self.prepare_table_data(
return_column_names=False,
rows_head=rows_head,
rows_tail=rows_tail,
max_row_height=max_row_height,
max_cell_length=max_cell_length,
)
for row in data:
rich_table.add_row(*row)
return rich_table
prepare_table_data(self, return_column_names=False, rows_head=None, rows_tail=None, max_row_height=None, max_cell_length=None)
¶Source code in kiara/utils/output.py
def prepare_table_data(
self,
return_column_names: bool = False,
rows_head: Union[int, None] = None,
rows_tail: Union[int, None] = None,
max_row_height: Union[int, None] = None,
max_cell_length: Union[int, None] = None,
) -> Iterator[Iterable[Any]]:
if return_column_names:
yield self.column_names
num_split_rows = 2
if rows_head is not None:
if rows_head < 0:
rows_head = 0
if rows_head > self.num_rows:
rows_head = self.num_rows
rows_tail = None
num_split_rows = 0
if rows_tail is not None:
if rows_head + rows_tail >= self.num_rows: # type: ignore
rows_head = self.num_rows
rows_tail = None
num_split_rows = 0
else:
num_split_rows = 0
if rows_head is not None:
head = self.slice(0, rows_head)
num_rows = rows_head
else:
head = self
num_rows = self.num_rows
table_dict = head.to_pydict()
for i in range(0, num_rows):
row = []
for cn in self.column_names:
cell = table_dict[cn][i]
cell_str = str(cell)
if max_row_height and max_row_height > 0 and "\n" in cell_str:
lines = cell_str.split("\n")
if len(lines) > max_row_height:
if max_row_height == 1:
lines = lines[0:1]
else:
half = int(max_row_height / 2)
lines = lines[0:half] + [".."] + lines[-half:]
cell_str = "\n".join(lines)
if max_cell_length and max_cell_length > 0:
lines = []
for line in cell_str.split("\n"):
if len(line) > max_cell_length:
line = line[0:max_cell_length] + " ..."
else:
line = line
lines.append(line)
cell_str = "\n".join(lines)
row.append(cell_str)
yield row
if num_split_rows:
for i in range(0, num_split_rows):
row = []
for _ in self.column_names:
row.append("...")
yield row
if rows_head:
if rows_tail is not None:
if rows_tail < 0:
rows_tail = 0
tail = self.slice(self.num_rows - rows_tail)
table_dict = tail.to_pydict()
for i in range(0, num_rows):
row = []
for cn in self.column_names:
cell = table_dict[cn][i]
cell_str = str(cell)
if max_row_height and max_row_height > 0 and "\n" in cell_str:
lines = cell_str.split("\n")
if len(lines) > max_row_height:
if max_row_height == 1:
lines = lines[0:1]
else:
half = int(len(lines) / 2)
lines = lines[0:half] + [".."] + lines[-half:]
cell_str = "\n".join(lines)
if max_cell_length and max_cell_length > 0:
lines = []
for line in cell_str.split("\n"):
if len(line) > max_cell_length:
line = line[0:(max_cell_length)] + " ..."
else:
line = line
lines.append(line)
cell_str = "\n".join(lines)
row.append(cell_str)
yield row
return
retrieve_column_names(self)
¶Source code in kiara/utils/output.py
@abstractmethod
def retrieve_column_names(self) -> Iterable[str]:
pass
retrieve_number_of_rows(self)
¶Source code in kiara/utils/output.py
@abstractmethod
def retrieve_number_of_rows(self) -> int:
pass
slice(self, offset=0, length=None)
¶Source code in kiara/utils/output.py
@abstractmethod
def slice(self, offset: int = 0, length: Union[int, None] = None) -> "TabularWrap":
pass
to_pydict(self)
¶Source code in kiara/utils/output.py
@abstractmethod
def to_pydict(self) -> Mapping:
pass
Functions¶
create_pipeline_steps_tree(pipeline_structure, pipeline_details)
¶Source code in kiara/utils/output.py
def create_pipeline_steps_tree(
pipeline_structure: "PipelineStructure", pipeline_details: "PipelineDetails"
) -> Tree:
from kiara.models.module.pipeline import StepStatus
steps = Tree("steps")
for idx, stage in enumerate(pipeline_structure.processing_stages, start=1):
stage_node = steps.add(f"stage: [i]{idx}[/i]")
for step_id in sorted(stage):
step_node = stage_node.add(f"step: [i]{step_id}[/i]")
step_details = pipeline_details.step_states[step_id]
status = step_details.status
if status is StepStatus.INPUTS_READY:
step_node.add("status: [yellow]inputs ready[/yellow]")
elif status is StepStatus.RESULTS_READY:
step_node.add("status: [green]results ready[/green]")
else:
invalid_node = step_node.add("status: [red]inputs invalid[/red]")
invalid = step_details.invalid_details
for k, v in invalid.items():
invalid_node.add(f"[i]{k}[/i]: {v}")
return steps
create_recursive_table_from_model_object(model, render_config=None)
¶Source code in kiara/utils/output.py
def create_recursive_table_from_model_object(
model: BaseModel,
render_config: Union[Mapping[str, Any], None] = None,
):
if render_config is None:
render_config = {}
show_lines = render_config.get("show_lines", True)
show_header = render_config.get("show_header", True)
model_cls = model.__class__
table = RichTable(box=box.SIMPLE, show_lines=show_lines, show_header=show_header)
table.add_column("Field")
table.add_column("Value")
props = model_cls.schema().get("properties", {})
for field_name in sorted(model_cls.__fields__.keys()):
data = getattr(model, field_name)
p = props.get(field_name, None)
p_type = None
if p is not None:
p_type = p.get("type", None)
# TODO: check 'anyOf' keys
if p_type is not None:
p_type = f"[i]{p_type}[/i]"
desc = p.get("description", None)
if not isinstance(data, BaseModel):
data_renderable = extract_renderable(data, render_config=render_config)
sub_model = None
else:
sub_model = create_recursive_table_from_model_object(
data, render_config={"show_lines": True, "show_header": False}
)
data_renderable = None
group = []
if data_renderable:
group.append(data_renderable)
group.append("")
if desc:
group.append(f"[i]{desc}[/i]")
if sub_model:
group.append(sub_model)
if p_type:
field_name = f"[b i]{field_name}[/b i] ([i]{p_type}[/i])"
else:
field_name = f"[b i]{field_name}[/b i]"
table.add_row(field_name, Group(*group))
return table
create_renderable_from_values(values, config=None)
¶Create a renderable for this module configuration.
Source code in kiara/utils/output.py
def create_renderable_from_values(
values: Mapping[str, "Value"], config: Union[Mapping[str, Any], None] = None
) -> RenderableType:
"""Create a renderable for this module configuration."""
if config is None:
config = {}
render_format = config.get("render_format", "terminal")
if render_format not in ["terminal"]:
raise Exception(f"Invalid render format: {render_format}")
show_pedigree = config.get("show_pedigree", False)
show_data = config.get("show_data", False)
show_hash = config.get("show_hash", True)
# show_load_config = config.get("show_load_config", False)
table = RichTable(show_lines=True, box=box.MINIMAL_DOUBLE_HEAD)
table.add_column("value_id", "i")
table.add_column("data_type")
table.add_column("size")
if show_hash:
table.add_column("hash")
if show_pedigree:
table.add_column("pedigree")
if show_data:
table.add_column("data")
for id, value in sorted(values.items(), key=lambda item: item[1].value_schema.type):
row: List[RenderableType] = [id, value.value_schema.type, str(value.value_size)]
if show_hash:
row.append(str(value.value_hash))
if show_pedigree:
if value.pedigree == ORPHAN:
pedigree = "-- n/a --"
else:
pedigree = value.pedigree.json(option=orjson.OPT_INDENT_2)
row.append(pedigree)
if show_data:
data = value._data_registry.pretty_print_data(
value_id=value.value_id, target_type="terminal_renderable", **config
)
row.append(data)
# if show_load_config:
# load_config = value.retrieve_load_config()
# if load_config is None:
# load_config_str: RenderableType = "-- not stored (yet) --"
# else:
# load_config_str = load_config.create_renderable()
# row.append(load_config_str)
table.add_row(*row)
return table
create_table_from_base_model_cls(model_cls)
¶Source code in kiara/utils/output.py
def create_table_from_base_model_cls(model_cls: Type[BaseModel]):
table = RichTable(box=box.SIMPLE, show_lines=True)
table.add_column("Field")
table.add_column("Type")
table.add_column("Description")
table.add_column("Required")
table.add_column("Default")
props = model_cls.schema().get("properties", {})
for field_name, field in sorted(model_cls.__fields__.items()):
row = [field_name]
p = props.get(field_name, None)
p_type = None
if p is not None:
p_type = p.get("type", None)
# TODO: check 'anyOf' keys
if p_type is None:
p_type = "-- check source --"
row.append(p_type)
desc = p.get("description", "")
row.append(desc)
row.append("yes" if field.required else "no")
default = field.default
if callable(default):
default = default()
if default is None:
default = ""
else:
try:
default = json.dumps(default, indent=2)
except Exception:
default = str(default)
row.append(default)
table.add_row(*row)
return table
create_table_from_field_schemas(fields, _add_default=True, _add_required=True, _show_header=False, _constants=None)
¶Source code in kiara/utils/output.py
def create_table_from_field_schemas(
fields: Mapping[str, "ValueSchema"],
_add_default: bool = True,
_add_required: bool = True,
_show_header: bool = False,
_constants: Union[Mapping[str, Any], None] = None,
) -> RichTable:
table = RichTable(box=box.SIMPLE, show_header=_show_header)
table.add_column("field name", style="i", overflow="fold")
table.add_column("type")
table.add_column("description")
if _add_required:
table.add_column("Required")
if _add_default:
if _constants:
table.add_column("Default / Constant")
else:
table.add_column("Default")
for field_name, schema in fields.items():
row: List[RenderableType] = [field_name, schema.type, schema.doc]
if _add_required:
req = schema.is_required()
if not req:
req_str = "no"
else:
if schema.default in [
None,
SpecialValue.NO_VALUE,
SpecialValue.NOT_SET,
]:
req_str = "[b]yes[b]"
else:
req_str = "no"
row.append(req_str)
if _add_default:
if _constants and field_name in _constants.keys():
d = f"[b]{_constants[field_name]}[/b] (constant)"
else:
if schema.default in [
None,
SpecialValue.NO_VALUE,
SpecialValue.NOT_SET,
]:
d = "-- no default --"
else:
d = str(schema.default)
row.append(d)
table.add_row(*row)
return table
create_table_from_model_object(model, render_config=None, exclude_fields=None)
¶Source code in kiara/utils/output.py
def create_table_from_model_object(
model: BaseModel,
render_config: Union[Mapping[str, Any], None] = None,
exclude_fields: Union[Set[str], None] = None,
):
model_cls = model.__class__
table = RichTable(box=box.SIMPLE, show_lines=True)
table.add_column("Field")
table.add_column("Type")
table.add_column("Value")
table.add_column("Description")
props = model_cls.schema().get("properties", {})
for field_name, field in sorted(model_cls.__fields__.items()):
if exclude_fields and field_name in exclude_fields:
continue
row: List[RenderableType] = [field_name]
p = props.get(field_name, None)
p_type = None
if p is not None:
p_type = p.get("type", None)
# TODO: check 'anyOf' keys
if p_type is None:
p_type = "-- check source --"
row.append(p_type)
data = getattr(model, field_name)
row.append(extract_renderable(data, render_config=render_config))
desc = p.get("description", "")
row.append(desc)
table.add_row(*row)
return table
create_value_map_status_renderable(inputs, render_config=None, fields=None)
¶Source code in kiara/utils/output.py
def create_value_map_status_renderable(
inputs: ValueMap,
render_config: Union[Mapping[str, Any], None] = None,
fields: Union[None, Iterable[str]] = None,
) -> RichTable:
if render_config is None:
render_config = {}
show_description: bool = render_config.get("show_description", True)
show_type: bool = render_config.get("show_type", True)
show_required: bool = render_config.get("show_required", True)
show_default: bool = render_config.get("show_default", True)
show_value_ids: bool = render_config.get("show_value_ids", False)
table = RichTable(box=box.SIMPLE, show_header=True)
table.add_column("field name", style="i")
table.add_column("status", style="b")
if show_type:
table.add_column("type")
if show_description:
table.add_column("description")
if show_required:
table.add_column("required")
if show_default:
table.add_column("default")
if show_value_ids:
table.add_column("value id", overflow="fold")
invalid = inputs.check_invalid()
if fields:
field_order = fields
else:
field_order = sorted(inputs.keys())
for field_name in field_order:
value = inputs.get(field_name, None)
if value is None:
log.debug(
"ignore.field", field_name=field_name, available_fields=inputs.keys()
)
continue
row: List[RenderableType] = [field_name]
if field_name in invalid.keys():
row.append(f"[red]{invalid[field_name]}[/red]")
else:
row.append("[green]valid[/green]")
value_schema = inputs.values_schema[field_name]
if show_type:
row.append(value_schema.type)
if show_description:
row.append(value_schema.doc.description)
if show_required:
req = value_schema.is_required()
if not req:
req_str = "no"
else:
if value_schema.default in [
None,
SpecialValue.NO_VALUE,
SpecialValue.NOT_SET,
]:
req_str = "[b]yes[b]"
else:
req_str = "no"
row.append(req_str)
if show_default:
default = value_schema.default
if callable(default):
default_val = default()
else:
default_val = default
if default_val in [None, SpecialValue.NOT_SET, SpecialValue.NO_VALUE]:
default_str = ""
else:
default_str = str(default_val)
row.append(default_str)
if show_value_ids:
row.append(str(inputs.get_value_obj(field_name=field_name).value_id))
table.add_row(*row)
return table
extract_renderable(item, render_config=None)
¶Try to automatically find and extract or create an object that is renderable by the 'rich' library.
Source code in kiara/utils/output.py
def extract_renderable(
item: Any, render_config: Union[Mapping[str, Any], None] = None
) -> RenderableType:
"""Try to automatically find and extract or create an object that is renderable by the 'rich' library."""
if render_config is None:
render_config = {}
else:
render_config = dict(render_config)
inline_models_as_json = render_config.setdefault("inline_models_as_json", True)
if hasattr(item, "create_renderable"):
return item.create_renderable(**render_config)
elif isinstance(item, (ConsoleRenderable, RichCast, str)):
return item
elif isinstance(item, BaseModel) and not inline_models_as_json:
return create_table_from_model_object(item)
elif isinstance(item, BaseModel):
return item.json(indent=2)
elif isinstance(item, Mapping) and not inline_models_as_json:
table = RichTable(show_header=False, box=box.SIMPLE)
table.add_column("Key", style="i")
table.add_column("Value")
for k, v in item.items():
table.add_row(k, extract_renderable(v, render_config=render_config))
return table
elif isinstance(item, Mapping):
result = {}
for k, v in item.items():
if isinstance(v, BaseModel):
v = v.dict()
result[k] = v
return orjson_dumps(
result, option=orjson.OPT_INDENT_2 | orjson.OPT_NON_STR_KEYS
)
elif isinstance(item, Iterable):
_all = []
for i in item:
_all.append(extract_renderable(i))
rg = Group(*_all)
return rg
elif isinstance(item, Enum):
return item.value
else:
return str(item)
pipelines
¶
Functions¶
check_doc_sidecar(path, data)
¶Source code in kiara/utils/pipelines.py
def check_doc_sidecar(
path: Union[Path, str], data: Mapping[str, Any]
) -> Mapping[str, Any]:
if isinstance(path, str):
path = Path(os.path.expanduser(path))
_doc = data["data"].get("documentation", None)
if _doc is None:
_doc_path = Path(path.as_posix() + ".md")
if _doc_path.is_file():
doc = _doc_path.read_text()
if doc:
data["data"]["documentation"] = doc
return data
create_step_value_address(value_address_config, default_field_name)
¶Source code in kiara/utils/pipelines.py
def create_step_value_address(
value_address_config: Union[str, Mapping[str, Any]],
default_field_name: str,
) -> "StepValueAddress":
if isinstance(value_address_config, StepValueAddress):
return value_address_config
sub_value: Union[Mapping[str, Any], None] = None
if isinstance(value_address_config, str):
tokens = value_address_config.split(".")
if len(tokens) == 1:
step_id = value_address_config
output_name = default_field_name
elif len(tokens) == 2:
step_id = tokens[0]
output_name = tokens[1]
elif len(tokens) == 3:
step_id = tokens[0]
output_name = tokens[1]
sub_value = {"config": tokens[2]}
else:
raise NotImplementedError()
elif isinstance(value_address_config, Mapping):
step_id = value_address_config["step_id"]
output_name = value_address_config["value_name"]
sub_value = value_address_config.get("sub_value", None)
else:
raise TypeError(
f"Invalid type for creating step value address: {type(value_address_config)}"
)
if sub_value is not None and not isinstance(sub_value, Mapping):
raise ValueError(
f"Invalid type '{type(sub_value)}' for sub_value (step_id: {step_id}, value name: {output_name}): {sub_value}"
)
input_link = StepValueAddress(
step_id=step_id, value_name=output_name, sub_value=sub_value
)
return input_link
ensure_step_value_addresses(link, default_field_name)
¶Source code in kiara/utils/pipelines.py
def ensure_step_value_addresses(
link: Union[str, Mapping, Iterable], default_field_name: str
) -> List["StepValueAddress"]:
if isinstance(link, (str, Mapping)):
input_links: List[StepValueAddress] = [
create_step_value_address(
value_address_config=link, default_field_name=default_field_name
)
]
elif isinstance(link, Iterable):
input_links = []
for o in link:
il = create_step_value_address(
value_address_config=o, default_field_name=default_field_name
)
input_links.append(il)
else:
raise TypeError(f"Can't parse input map, invalid type for output: {link}")
return input_links
get_pipeline_details_from_path(path, module_type_name=None, base_module=None)
¶Load a pipeline description, save it's content, and determine it the pipeline base name.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
Union[str, pathlib.Path] |
the path to the pipeline file |
required |
module_type_name |
Optional[str] |
if specifies, overwrites any auto-detected or assigned pipeline name |
None |
base_module |
Optional[str] |
overrides the base module the assembled pipeline module will be located in the python hierarchy |
None |
Source code in kiara/utils/pipelines.py
def get_pipeline_details_from_path(
path: Union[str, Path],
module_type_name: Union[str, None] = None,
base_module: Union[str, None] = None,
) -> Mapping[str, Any]:
"""Load a pipeline description, save it's content, and determine it the pipeline base name.
Arguments:
path: the path to the pipeline file
module_type_name: if specifies, overwrites any auto-detected or assigned pipeline name
base_module: overrides the base module the assembled pipeline module will be located in the python hierarchy
"""
if isinstance(path, str):
path = Path(os.path.expanduser(path))
if not path.is_file():
raise Exception(
f"Can't add pipeline description '{path.as_posix()}': not a file"
)
data = get_data_from_file(path)
if not data:
raise Exception(
f"Can't register pipeline file '{path.as_posix()}': no content."
)
if module_type_name:
data[MODULE_TYPE_NAME_KEY] = module_type_name
if not isinstance(data, Mapping):
raise Exception("Not a dictionary type.")
# filename = path.name
# name = data.get(MODULE_TYPE_NAME_KEY, None)
# if name is None:
# name = filename.split(".", maxsplit=1)[0]
result = {"data": data, "source": path.as_posix(), "source_type": "file"}
if base_module:
result["base_module"] = base_module
return result
string_vars
¶
log
¶create_var_regex(delimiter_start=None, delimiter_end=None)
¶Source code in kiara/utils/string_vars.py
def create_var_regex(
delimiter_start: Union[str, None] = None, delimiter_end: Union[str, None] = None
) -> Pattern:
if delimiter_start is None:
delimiter_start = "\\$\\{"
# TODO: make this smarter
if delimiter_end is None:
delimiter_end = "\\}"
regex = re.compile(delimiter_start + "\\s*(.+?)\\s*" + delimiter_end)
return regex
find_regex_matches_in_obj(source_obj, regex, current=None)
¶Source code in kiara/utils/string_vars.py
def find_regex_matches_in_obj(
source_obj: Any, regex: Pattern, current: Union[Set[str], None] = None
) -> Set[str]:
if current is None:
current = set()
if not source_obj:
return current
if isinstance(source_obj, Mapping):
for k, v in source_obj.items():
find_regex_matches_in_obj(k, regex=regex, current=current)
find_regex_matches_in_obj(v, regex=regex, current=current)
elif isinstance(source_obj, str):
matches = regex.findall(source_obj)
current.update(matches)
elif isinstance(source_obj, Sequence):
for item in source_obj:
find_regex_matches_in_obj(item, regex=regex, current=current)
return current
find_var_names_in_obj(template_obj, delimiter=None, delimiter_end=None)
¶Source code in kiara/utils/string_vars.py
def find_var_names_in_obj(
template_obj: Any,
delimiter: Union[Pattern, str, None] = None,
delimiter_end: Union[str, None] = None,
) -> Set[str]:
if isinstance(delimiter, Pattern):
regex = delimiter
else:
regex = create_var_regex(delimiter_start=delimiter, delimiter_end=delimiter_end)
var_names = find_regex_matches_in_obj(template_obj, regex=regex)
return var_names
replace_var_names_in_obj(template_obj, repl_dict, delimiter=None, delimiter_end=None, ignore_missing_keys=False)
¶Source code in kiara/utils/string_vars.py
def replace_var_names_in_obj(
template_obj: Any,
repl_dict: typing.Mapping[str, Any],
delimiter: Union[Pattern, str, None] = None,
delimiter_end: Union[str, None] = None,
ignore_missing_keys: bool = False,
) -> Any:
if isinstance(delimiter, Pattern):
regex = delimiter
else:
regex = create_var_regex(delimiter_start=delimiter, delimiter_end=delimiter_end)
if not template_obj:
return template_obj
if isinstance(template_obj, Mapping):
result: Any = {}
for k, v in template_obj.items():
key = replace_var_names_in_obj(
template_obj=k,
repl_dict=repl_dict,
delimiter=regex,
ignore_missing_keys=ignore_missing_keys,
)
value = replace_var_names_in_obj(
template_obj=v,
repl_dict=repl_dict,
delimiter=regex,
ignore_missing_keys=ignore_missing_keys,
)
result[key] = value
elif isinstance(template_obj, str):
result = replace_var_names_in_string(
template_obj,
repl_dict=repl_dict,
regex=regex,
ignore_missing_keys=ignore_missing_keys,
)
elif isinstance(template_obj, Sequence):
result = []
for item in template_obj:
r = replace_var_names_in_obj(
item,
repl_dict=repl_dict,
delimiter=regex,
ignore_missing_keys=ignore_missing_keys,
)
result.append(r)
else:
result = template_obj
return result
replace_var_names_in_string(template_string, repl_dict, regex, ignore_missing_keys=False)
¶Source code in kiara/utils/string_vars.py
def replace_var_names_in_string(
template_string: str,
repl_dict: typing.Mapping[str, Any],
regex: Pattern,
ignore_missing_keys: bool = False,
) -> str:
def sub(match):
key = match.groups()[0]
if key not in repl_dict.keys():
if not ignore_missing_keys:
raise Exception(
msg=f"Can't insert variable '{key}'. Key not in provided input values, available keys: {', '.join(repl_dict.keys())}",
)
else:
return match[0]
else:
result = repl_dict[key]
return result
result = regex.sub(sub, template_string)
return result
values
¶
augment_values(values, schemas, constants=None)
¶Source code in kiara/utils/values.py
def augment_values(
values: Mapping[str, Any],
schemas: Mapping[str, ValueSchema],
constants: Union[Mapping[str, ValueSchema], None] = None,
) -> Dict[str, Any]:
# TODO: check if extra fields were provided
if constants:
for k, v in constants.items():
if k in values.keys():
raise Exception(f"Invalid input: value provided for constant '{k}'")
values_new = {}
if constants:
for field_name, schema in constants.items():
v = schema.default
assert v not in [None, SpecialValue.NO_VALUE, SpecialValue.NOT_SET]
if callable(v):
values_new[field_name] = v()
else:
values_new[field_name] = copy.deepcopy(v)
for field_name, schema in schemas.items():
if field_name in values_new.keys():
raise Exception(
f"Duplicate field '{field_name}', this is most likely a bug."
)
if field_name not in values.keys():
if schema.default != SpecialValue.NOT_SET:
if callable(schema.default):
values_new[field_name] = schema.default()
else:
values_new[field_name] = copy.deepcopy(schema.default)
else:
values_new[field_name] = SpecialValue.NOT_SET
else:
value = values[field_name]
if value is None:
value = SpecialValue.NO_VALUE
values_new[field_name] = value
return values_new
create_schema_dict(schema_config)
¶Source code in kiara/utils/values.py
def create_schema_dict(
schema_config: Mapping[str, Union[ValueSchema, Mapping[str, Any]]],
) -> Mapping[str, ValueSchema]:
invalid = check_valid_field_names(*schema_config.keys())
if invalid:
raise Exception(
f"Can't assemble schema because it contains invalid input field name(s) '{', '.join(invalid)}'. Change the input schema to not contain any of the reserved keywords: {', '.join(INVALID_VALUE_NAMES)}"
)
result = {}
for k, v in schema_config.items():
if isinstance(v, ValueSchema):
result[k] = v
elif isinstance(v, Mapping):
_v = dict(v)
if "doc" not in _v.keys():
_v["doc"] = DEFAULT_NO_DESC_VALUE
schema = ValueSchema(**_v)
result[k] = schema
else:
if v is None:
msg = "None"
else:
msg = v.__class__
raise Exception(
f"Invalid return type '{msg}' for field '{k}' when trying to create schema."
)
return result
extract_raw_value(kiara, value_id)
¶Source code in kiara/utils/values.py
def extract_raw_value(kiara: "Kiara", value_id: uuid.UUID):
value = kiara.data_registry.get_value(value=value_id)
# TODO: check without import
from kiara.models.values.value import ORPHAN
if value.pedigree != ORPHAN:
# TODO: find alias?
return f'"value:{value_id}"'
else:
if value.value_schema.type == "string":
return f'"{value.data}"'
elif value.value_schema.type == "list":
return value.data.list_data
else:
return value.data
extract_raw_values(kiara, **value_ids)
¶Source code in kiara/utils/values.py
def extract_raw_values(kiara: "Kiara", **value_ids: uuid.UUID) -> Dict[str, Any]:
result = {}
for field_name, value_id in value_ids.items():
result[field_name] = extract_raw_value(kiara=kiara, value_id=value_id)
return result
overlay_constants_and_defaults(schemas, defaults, constants)
¶Source code in kiara/utils/values.py
def overlay_constants_and_defaults(
schemas: Mapping[str, ValueSchema],
defaults: Mapping[str, Any],
constants: Mapping[str, Any],
):
for k, v in schemas.items():
default_value = defaults.get(k, None)
constant_value = constants.get(k, None)
# value_to_test = None
if default_value is not None and constant_value is not None:
raise Exception(
f"Module configuration error. Value '{k}' set in both 'constants' and 'defaults', this is not allowed."
)
# TODO: perform validation for constants/defaults
if default_value is not None:
schemas[k].default = default_value
if constant_value is not None:
schemas[k].default = constant_value
schemas[k].is_constant = True
input_schemas = {}
constants = {}
for k, v in schemas.items():
if v.is_constant:
constants[k] = v
else:
input_schemas[k] = v
return input_schemas, constants
yaml
¶
StringYAML (YAML)
¶Source code in kiara/utils/yaml.py
class StringYAML(YAML):
def dump(self, data, stream=None, **kw):
inefficient = False
if stream is None:
inefficient = True
stream = StringIO()
YAML.dump(self, data, stream, **kw)
if inefficient:
return stream.getvalue()
dump(self, data, stream=None, **kw)
¶Source code in kiara/utils/yaml.py
def dump(self, data, stream=None, **kw):
inefficient = False
if stream is None:
inefficient = True
stream = StringIO()
YAML.dump(self, data, stream, **kw)
if inefficient:
return stream.getvalue()